diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 029e6c09..02c7c1dd 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -3,18 +3,25 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_floats; +use crate::function::functions_nested_utils::list_element_field; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; use arrow::buffer::{BooleanBuffer, NullBuffer}; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{Result, exec_err}; +use datafusion_expr::ReturnFieldArgs; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_functions_nested::array_has::array_has_udf; use std::sync::Arc; +/// Spark's `ArrayContains` is a null-intolerant `BinaryExpression`. It is also nullable +/// when the array can hold NULL elements. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `array_contains` function. /// /// Calls DataFusion's `array_has` and then applies Spark's null semantics: @@ -59,6 +66,19 @@ impl ScalarUDFImpl for SparkArrayContains { Ok(DataType::Boolean) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let contains_null = list_element_field(args.arg_fields[0].data_type()) + .is_none_or(|field| field.is_nullable()); + let nullable = NULLABILITY.nullable(args.arg_fields) || contains_null; + Ok(Arc::new(Field::new( + self.name(), + DataType::Boolean, + nullable, + ))) + } + + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, mut args: ScalarFunctionArgs) -> Result { for arg in &mut args.args { *arg = normalize_floats(arg)?; @@ -161,9 +181,30 @@ fn mask_with_list_nulls( mod tests { use super::*; use arrow::array::{FixedSizeListArray, Float64Array}; - use arrow::datatypes::Field; use datafusion_common::ScalarValue; + fn return_field_nullable(array_nullable: bool, element_nullable: bool) -> bool { + let element = Arc::new(Field::new("item", DataType::Int64, element_nullable)); + let arg_fields = [ + Arc::new(Field::new("a", DataType::List(element), array_nullable)), + Arc::new(Field::new("v", DataType::Int64, false)), + ]; + SparkArrayContains::new() + .return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &[None, None], + }) + .unwrap() + .is_nullable() + } + + #[test] + fn return_field_is_nullable_when_the_array_can_hold_nulls() { + assert!(!return_field_nullable(false, false)); + assert!(return_field_nullable(true, false)); + assert!(return_field_nullable(false, true)); + } + #[test] fn zero_width_fixed_size_list_keeps_its_row() { // One valid row of a zero-width list, without a null buffer to infer the length from. diff --git a/crates/spark/src/function/array/array_extrema.rs b/crates/spark/src/function/array/array_extrema.rs index 28cdbad4..00d313c1 100644 --- a/crates/spark/src/function/array/array_extrema.rs +++ b/crates/spark/src/function/array/array_extrema.rs @@ -5,6 +5,7 @@ //! Spark-compatible `array_max` and `array_min`. use crate::function::float_normalize::normalize_array; +use crate::function::nullability::{SparkNullability, spark_nullability}; use crate::function::type_coercion::{is_orderable, spark_sql_type}; use arrow::array::{ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray, OffsetSizeTrait, @@ -26,6 +27,10 @@ use datafusion_macros::user_doc; use std::cmp::Ordering; use std::sync::Arc; +/// Spark's `ArrayMax` and `ArrayMin` are null-intolerant and always nullable, because an +/// empty array returns NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `array_max` expression. /// /// @@ -82,6 +87,8 @@ impl ScalarUDFImpl for SparkArrayMax { element_type(self.name(), arg_types) } + spark_nullability!(NULLABILITY, return_type); + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { coerce_array(self.name(), arg_types) } @@ -151,6 +158,8 @@ impl ScalarUDFImpl for SparkArrayMin { element_type(self.name(), arg_types) } + spark_nullability!(NULLABILITY, return_type); + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { coerce_array(self.name(), arg_types) } diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs index 28002d03..5862106e 100644 --- a/crates/spark/src/function/array/array_insert.rs +++ b/crates/spark/src/function/array/array_insert.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::functions_nested_utils::list_element_field; use std::sync::Arc; use arrow::array::{ @@ -103,16 +104,6 @@ impl SparkArrayInsert { } } -/// The element field of a list type. -fn list_element_field(data_type: &DataType) -> Option<&FieldRef> { - match data_type { - DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { - Some(f) - } - _ => None, - } -} - /// Spark's error when the first argument is not an array, or the item does /// not share a type with the array's elements. fn diff_types_err(array: &DataType, item: &DataType) -> DataFusionError { diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 445a8b48..6c5c5333 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_array; +use crate::function::nullability::{SparkNullability, spark_nullability}; use crate::function::type_coercion::{ is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; @@ -21,6 +22,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `ArrayPosition` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `array_position` expression. /// /// @@ -85,6 +89,8 @@ impl ScalarUDFImpl for SparkArrayPosition { Ok(DataType::Int64) } + spark_nullability!(NULLABILITY, return_type); + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { let [array, element] = take_function_args(self.name(), arg_types)?; coerce_array_and_element(self.name(), array, element) diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs index 77b1da94..125ee163 100644 --- a/crates/spark/src/function/array/array_remove.rs +++ b/crates/spark/src/function/array/array_remove.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_array; +use crate::function::nullability::{SparkNullability, spark_nullability}; use crate::function::type_coercion::{ is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; @@ -24,6 +25,9 @@ use datafusion_macros::user_doc; use num_traits::Float; use std::sync::Arc; +/// Spark's `ArrayRemove` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `array_remove` expression. /// /// @@ -85,6 +89,8 @@ impl ScalarUDFImpl for SparkArrayRemove { Ok(array.clone()) } + spark_nullability!(NULLABILITY, return_type); + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { let [array, element] = take_function_args(self.name(), arg_types)?; coerce_array_and_element(self.name(), array, element) diff --git a/crates/spark/src/function/array/slice.rs b/crates/spark/src/function/array/slice.rs index 5e6f7274..14234266 100644 --- a/crates/spark/src/function/array/slice.rs +++ b/crates/spark/src/function/array/slice.rs @@ -18,6 +18,11 @@ use datafusion_functions_nested::extract::array_slice_udf; use datafusion_macros::user_doc; use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `Slice` is a null-intolerant `TernaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `slice` expression. /// /// @@ -94,8 +99,6 @@ impl ScalarUDFImpl for SparkSlice { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - let data_type = match args.arg_fields[0].data_type() { DataType::Null => { DataType::List(Arc::new(Field::new_list_field(DataType::Null, true))) @@ -103,9 +106,11 @@ impl ScalarUDFImpl for SparkSlice { dt => dt.clone(), }; - Ok(Arc::new(Field::new("slice", data_type, nullable))) + Ok(NULLABILITY.return_field("slice", data_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/bitwise/bit_count.rs b/crates/spark/src/function/bitwise/bit_count.rs index 9c63274a..ce2b4ebf 100644 --- a/crates/spark/src/function/bitwise/bit_count.rs +++ b/crates/spark/src/function/bitwise/bit_count.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray, Int32Array}; use arrow::datatypes::{ DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, @@ -18,6 +19,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `BitwiseCount` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `bit_count` expression. /// /// @@ -91,15 +95,11 @@ impl ScalarUDFImpl for SparkBitCount { &self, args: datafusion_expr::ReturnFieldArgs, ) -> Result { - use arrow::datatypes::Field; - // bit_count returns Int32 with the same nullability as the input - Ok(Arc::new(Field::new( - args.arg_fields[0].name(), - DataType::Int32, - args.arg_fields[0].is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/bitwise/bit_get.rs b/crates/spark/src/function/bitwise/bit_get.rs index 64b6e50a..d1a15ef5 100644 --- a/crates/spark/src/function/bitwise/bit_get.rs +++ b/crates/spark/src/function/bitwise/bit_get.rs @@ -10,7 +10,7 @@ use arrow::array::{ downcast_integer_array, }; use arrow::compute::try_binary; -use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef, Int8Type, Int32Type}; +use arrow::datatypes::{ArrowNativeType, DataType, FieldRef, Int8Type, Int32Type}; use datafusion_common::types::{NativeType, logical_int32}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; @@ -21,6 +21,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `BitwiseGet` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `bit_get` expression. /// /// @@ -97,11 +102,11 @@ impl ScalarUDFImpl for SparkBitGet { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - // Spark derives nullability for BinaryExpression from its children - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Int8, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int8, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -142,6 +147,7 @@ fn spark_bit_get(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_bit_get_nullability_non_nullable_inputs() { diff --git a/crates/spark/src/function/bitwise/bit_shift.rs b/crates/spark/src/function/bitwise/bit_shift.rs index a50d7282..19917afa 100644 --- a/crates/spark/src/function/bitwise/bit_shift.rs +++ b/crates/spark/src/function/bitwise/bit_shift.rs @@ -4,11 +4,11 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, Int32Array, PrimitiveArray}; use arrow::compute; use arrow::datatypes::{ - ArrowNativeType, DataType, Field, FieldRef, Int32Type, Int64Type, UInt32Type, - UInt64Type, + ArrowNativeType, DataType, FieldRef, Int32Type, Int64Type, UInt32Type, UInt64Type, }; use datafusion_common::types::{ NativeType, logical_int8, logical_int16, logical_int32, logical_int64, logical_uint8, @@ -180,6 +180,10 @@ enum BitShiftType { RightUnsigned, } +/// Spark's `ShiftLeft`, `ShiftRight` and `ShiftRightUnsigned` are null-intolerant +/// `BinaryExpression`s. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `shiftleft`, `shiftright` and `shiftrightunsigned` expressions. /// /// One struct implements all three functions, so each function's documentation is @@ -353,11 +357,12 @@ impl ScalarUDFImpl for SparkBitShift { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); let data_type = args.arg_fields[0].data_type().clone(); - Ok(Arc::new(Field::new(self.name(), data_type, nullable))) + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { match self.bit_shift_type { BitShiftType::Left => ShiftLeftDoc.doc(), @@ -377,6 +382,7 @@ impl ScalarUDFImpl for SparkBitShift { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_bit_shift_nullability() -> Result<()> { diff --git a/crates/spark/src/function/bitwise/bitwise_not.rs b/crates/spark/src/function/bitwise/bitwise_not.rs index 758fc321..6b6d141e 100644 --- a/crates/spark/src/function/bitwise/bitwise_not.rs +++ b/crates/spark/src/function/bitwise/bitwise_not.rs @@ -2,11 +2,10 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::*; use arrow::compute::kernels::bitwise; -use arrow::datatypes::{ - DataType, Field, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type, -}; +use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type}; use datafusion_common::{Result, internal_err, plan_err}; use datafusion_expr::{ColumnarValue, Documentation, TypeSignature, Volatility}; use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature}; @@ -14,6 +13,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `BitwiseNot` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `bitwise_not` expression (the `~` operator in Spark SQL). /// /// @@ -79,13 +81,12 @@ impl ScalarUDFImpl for SparkBitwiseNot { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - Ok(Arc::new(Field::new( - self.name(), - args.arg_fields[0].data_type().clone(), - args.arg_fields[0].is_nullable(), - ))) + let data_type = args.arg_fields[0].data_type().clone(); + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -133,6 +134,7 @@ pub fn spark_bitwise_not(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; use std::sync::Arc; #[test] diff --git a/crates/spark/src/function/datetime/add_months.rs b/crates/spark/src/function/datetime/add_months.rs index c8d28789..8653e687 100644 --- a/crates/spark/src/function/datetime/add_months.rs +++ b/crates/spark/src/function/datetime/add_months.rs @@ -3,9 +3,9 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use std::ops::Add; -use std::sync::Arc; -use arrow::datatypes::{DataType, Field, FieldRef, IntervalUnit}; +use crate::function::nullability::SparkNullability; +use arrow::datatypes::{DataType, FieldRef, IntervalUnit}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; @@ -15,6 +15,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `AddMonths` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `add_months` expression. /// /// @@ -81,13 +84,7 @@ impl ScalarUDFImpl for SparkAddMonths { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/datetime/date_add.rs b/crates/spark/src/function/datetime/date_add.rs index acec7759..959f2174 100644 --- a/crates/spark/src/function/datetime/date_add.rs +++ b/crates/spark/src/function/datetime/date_add.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use arrow::array::ArrayRef; use arrow::compute; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef}; +use arrow::datatypes::{DataType, Date32Type, FieldRef}; use datafusion_common::cast::{ as_date32_array, as_int8_array, as_int16_array, as_int32_array, }; @@ -19,6 +19,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `DateAdd` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `date_add` expression. /// /// @@ -95,14 +100,11 @@ impl ScalarUDFImpl for SparkDateAdd { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -153,6 +155,7 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_date_add_non_nullable_inputs() { diff --git a/crates/spark/src/function/datetime/date_diff.rs b/crates/spark/src/function/datetime/date_diff.rs index f593b566..e41b3d92 100644 --- a/crates/spark/src/function/datetime/date_diff.rs +++ b/crates/spark/src/function/datetime/date_diff.rs @@ -2,9 +2,8 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - -use arrow::datatypes::{DataType, Field, FieldRef}; +use crate::function::nullability::SparkNullability; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::types::{NativeType, logical_date, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; @@ -16,6 +15,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `DateDiff` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `date_diff` expression. /// /// @@ -104,8 +106,7 @@ impl ScalarUDFImpl for SparkDateDiff { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/datetime/date_from_unix_date.rs b/crates/spark/src/function/datetime/date_from_unix_date.rs index deed4a21..9871e011 100644 --- a/crates/spark/src/function/datetime/date_from_unix_date.rs +++ b/crates/spark/src/function/datetime/date_from_unix_date.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use arrow::array::{Array, Date32Array}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::as_int32_array; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; @@ -16,6 +16,10 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::int32_coercion; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `DateFromUnixDate` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `date_from_unix_date` expression. /// @@ -81,14 +85,11 @@ impl ScalarUDFImpl for SparkDateFromUnixDate { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields[0].is_nullable(); - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/datetime/date_sub.rs b/crates/spark/src/function/datetime/date_sub.rs index 3d5829f2..dc79641f 100644 --- a/crates/spark/src/function/datetime/date_sub.rs +++ b/crates/spark/src/function/datetime/date_sub.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use arrow::array::ArrayRef; use arrow::compute; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef}; +use arrow::datatypes::{DataType, Date32Type, FieldRef}; use datafusion_common::cast::{ as_date32_array, as_int8_array, as_int16_array, as_int32_array, }; @@ -18,6 +18,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `DateSub` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `date_sub` expression. /// /// @@ -87,14 +92,11 @@ impl ScalarUDFImpl for SparkDateSub { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -150,6 +152,7 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_date_sub_nullability_non_nullable_args() { diff --git a/crates/spark/src/function/datetime/dayname.rs b/crates/spark/src/function/datetime/dayname.rs index 48544a34..3842d6ba 100644 --- a/crates/spark/src/function/datetime/dayname.rs +++ b/crates/spark/src/function/datetime/dayname.rs @@ -2,10 +2,8 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - use arrow::compute::DatePart; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -15,6 +13,7 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::{date_part_name, date_signature}; +use crate::function::nullability::{SparkNullability, spark_nullability}; const DAY_NAMES: [&str; 7] = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]; @@ -23,6 +22,9 @@ fn day_number_to_name(day: i32) -> Option<&'static str> { DAY_NAMES.get(day as usize).copied() } +/// Spark's `DayName` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `dayname` expression (Spark 4.0+). /// /// @@ -84,10 +86,11 @@ impl ScalarUDFImpl for SparkDayName { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/datetime/extract.rs b/crates/spark/src/function/datetime/extract.rs index 200d621b..1a63ba6c 100644 --- a/crates/spark/src/function/datetime/extract.rs +++ b/crates/spark/src/function/datetime/extract.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::ArrayRef; use arrow::compute::{DatePart, date_part}; use arrow::datatypes::DataType; @@ -26,6 +27,9 @@ fn extract_signature() -> Signature { // SparkHour // ----------------------------------------------------------------------------- +/// Spark's `Hour`, `Minute` and `Second` are null-intolerant `UnaryExpression`s. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `hour` expression. /// /// @@ -78,6 +82,8 @@ impl ScalarUDFImpl for SparkHour { Ok(DataType::Int32) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -149,6 +155,8 @@ impl ScalarUDFImpl for SparkMinute { Ok(DataType::Int32) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -221,6 +229,8 @@ impl ScalarUDFImpl for SparkSecond { Ok(DataType::Int32) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/datetime/from_utc_timestamp.rs b/crates/spark/src/function/datetime/from_utc_timestamp.rs index f40dba59..0763130f 100644 --- a/crates/spark/src/function/datetime/from_utc_timestamp.rs +++ b/crates/spark/src/function/datetime/from_utc_timestamp.rs @@ -7,6 +7,7 @@ use std::hash::{Hash, Hasher}; use std::sync::Arc; use super::timezone::TimeZoneRules; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, ArrayRef, AsArray, BooleanArray, PrimitiveBuilder, StringArrayType, new_empty_array, @@ -28,6 +29,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; +/// Spark's `FromUTCTimestamp` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `from_utc_timestamp` expression. /// /// This UDF operates on already-evaluated arguments. Callers that need conditional @@ -113,15 +117,12 @@ impl ScalarUDFImpl for SparkFromUtcTimestamp { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - - Ok(Arc::new(Field::new( - self.name(), - args.arg_fields[0].data_type().clone(), - nullable, - ))) + let data_type = args.arg_fields[0].data_type().clone(); + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_from_utc_timestamp, vec![])(&args.args) } diff --git a/crates/spark/src/function/datetime/last_day.rs b/crates/spark/src/function/datetime/last_day.rs index 84029680..b260c5ae 100644 --- a/crates/spark/src/function/datetime/last_day.rs +++ b/crates/spark/src/function/datetime/last_day.rs @@ -4,8 +4,9 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray, Date32Array}; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef}; +use arrow::datatypes::{DataType, Date32Type, FieldRef}; use chrono::{Datelike, Duration, NaiveDate}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_datafusion_err, internal_err}; @@ -15,6 +16,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `LastDay` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `last_day` expression. /// /// @@ -67,17 +71,11 @@ impl ScalarUDFImpl for SparkLastDay { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let Some(field) = args.arg_fields.first() else { - return internal_err!("Spark `last_day` expects exactly one argument"); - }; - - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - field.is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -150,6 +148,7 @@ mod tests { use super::*; use crate::function::utils::test::test_scalar_function; use arrow::array::Array; + use arrow::datatypes::Field; #[test] fn test_last_day_nullability_matches_input() { diff --git a/crates/spark/src/function/datetime/make_date.rs b/crates/spark/src/function/datetime/make_date.rs index d8e020cb..b1bd7e94 100644 --- a/crates/spark/src/function/datetime/make_date.rs +++ b/crates/spark/src/function/datetime/make_date.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use arrow::array::{Array, Date32Array, Date32Builder, new_empty_array}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::as_int32_array; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; @@ -16,9 +16,16 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::int32_coercion; +use crate::function::nullability::{SparkNullability, spark_nullability}; const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. SQLSTATE: 22003"#; +/// Spark's `MakeDate` is null-intolerant. It is always nullable when ANSI mode is off, +/// because an invalid date returns NULL, and nullable when an argument is when ANSI mode is +/// on. ANSI mode is not known in `return_field_from_args`, so the result is always nullable +/// here. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `make_date` expression. /// /// @@ -97,12 +104,12 @@ impl ScalarUDFImpl for SparkMakeDate { internal_err!("return_field_from_args should be used instead") } - fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { - // An invalid date returns NULL when ANSI mode is off, which is not known - // at planning time, so the result is always nullable. - Ok(Arc::new(Field::new(self.name(), DataType::Date32, true))) + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -279,6 +286,7 @@ fn days_from_civil(year: i32, month: i32, day: i32) -> i64 { mod tests { use super::*; use arrow::array::Int32Array; + use arrow::datatypes::Field; use datafusion_common::config::ConfigOptions; fn invoke( diff --git a/crates/spark/src/function/datetime/make_dt_interval.rs b/crates/spark/src/function/datetime/make_dt_interval.rs index 87dc9d5f..450c7f37 100644 --- a/crates/spark/src/function/datetime/make_dt_interval.rs +++ b/crates/spark/src/function/datetime/make_dt_interval.rs @@ -4,11 +4,12 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, ArrayRef, AsArray, DurationMicrosecondBuilder, PrimitiveArray, }; use arrow::datatypes::TimeUnit::Microsecond; -use arrow::datatypes::{DataType, Field, FieldRef, Float64Type, Int32Type}; +use arrow::datatypes::{DataType, FieldRef, Float64Type, Int32Type}; use datafusion_common::types::{NativeType, logical_float64, logical_int32}; use datafusion_common::{ DataFusionError, Result, ScalarValue, internal_err, plan_datafusion_err, @@ -20,6 +21,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `MakeDTInterval` is a null-intolerant `QuaternaryExpression` that is nullable +/// when an argument is, but here an overflow returns NULL (#20), so the result is always +/// nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `make_dt_interval` expression. /// /// @@ -130,26 +136,15 @@ impl ScalarUDFImpl for SparkMakeDtInterval { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let has_non_finite_secs = args - .scalar_arguments - .get(3) - .and_then(|arg| { - arg.map(|scalar| match scalar { - ScalarValue::Float64(Some(v)) => !v.is_finite(), - ScalarValue::Float32(Some(v)) => !v.is_finite(), - _ => false, - }) - }) - .unwrap_or(false); - let nullable = - has_non_finite_secs || args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new( + Ok(NULLABILITY.return_field( self.name(), DataType::Duration(Microsecond), - nullable, - ))) + args.arg_fields, + )) } + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.is_empty() { return Ok(ColumnarValue::Scalar(ScalarValue::DurationMicrosecond( @@ -284,6 +279,7 @@ mod tests { use datafusion_common::internal_datafusion_err; use super::*; + use arrow::datatypes::Field; fn run_make_dt_interval(arrs: Vec) -> Result { make_dt_interval_kernel(&arrs) @@ -346,55 +342,21 @@ mod tests { } #[test] - fn return_field_respects_nullability() -> Result<()> { - let udf = SparkMakeDtInterval::new(); - - // All nullable inputs -> nullable output + fn return_field_is_always_nullable() -> Result<()> { + // An overflow returns NULL (#20), so the result is nullable even for + // non-nullable inputs let arg_fields = vec![ - Arc::new(Field::new("days", DataType::Int32, true)), - Arc::new(Field::new("hours", DataType::Int32, true)), - Arc::new(Field::new("mins", DataType::Int32, true)), - Arc::new(Field::new("secs", DataType::Float64, true)), - ]; - - let out = udf.return_field_from_args(ReturnFieldArgs { - arg_fields: &arg_fields, - scalar_arguments: &[None, None, None, None], - })?; - assert!(out.is_nullable()); - assert_eq!(out.data_type(), &Duration(Microsecond)); - - // Non-nullable inputs -> non-nullable output - let non_nullable_arg_fields = vec![ Arc::new(Field::new("days", DataType::Int32, false)), Arc::new(Field::new("hours", DataType::Int32, false)), Arc::new(Field::new("mins", DataType::Int32, false)), Arc::new(Field::new("secs", DataType::Float64, false)), ]; - - let out = udf.return_field_from_args(ReturnFieldArgs { - arg_fields: &non_nullable_arg_fields, + let out = SparkMakeDtInterval::new().return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, scalar_arguments: &[None, None, None, None], })?; - assert!(!out.is_nullable()); - - // Non-finite secs scalar should force nullable even if fields are non-nullable - let scalar_values = - [None, None, None, Some(ScalarValue::Float64(Some(f64::NAN)))]; - let scalar_refs = scalar_values.iter().map(|v| v.as_ref()).collect::>(); - let out = udf.return_field_from_args(ReturnFieldArgs { - arg_fields: &non_nullable_arg_fields, - scalar_arguments: &scalar_refs, - })?; assert!(out.is_nullable()); - - // Zero-arg call (defaults) should also be non-nullable - let out = udf.return_field_from_args(ReturnFieldArgs { - arg_fields: &[], - scalar_arguments: &[], - })?; - assert!(!out.is_nullable()); - + assert_eq!(out.data_type(), &Duration(Microsecond)); Ok(()) } diff --git a/crates/spark/src/function/datetime/make_interval.rs b/crates/spark/src/function/datetime/make_interval.rs index 88bfdaee..e262d60a 100644 --- a/crates/spark/src/function/datetime/make_interval.rs +++ b/crates/spark/src/function/datetime/make_interval.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, IntervalMonthDayNanoBuilder, PrimitiveArray}; use arrow::datatypes::DataType::Interval; use arrow::datatypes::IntervalUnit::MonthDayNano; @@ -17,6 +18,12 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `MakeInterval` is null-intolerant. It is always nullable when ANSI mode is off, +/// because an overflow returns NULL, and nullable when an argument is when ANSI mode is on. +/// ANSI mode is not known in `return_field_from_args`, so the result is always nullable +/// here. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `make_interval` expression. /// /// @@ -156,6 +163,8 @@ impl ScalarUDFImpl for SparkMakeInterval { Ok(Interval(MonthDayNano)) } + spark_nullability!(NULLABILITY, return_type); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.is_empty() { return Ok(ColumnarValue::Scalar(ScalarValue::IntervalMonthDayNano( diff --git a/crates/spark/src/function/datetime/monthname.rs b/crates/spark/src/function/datetime/monthname.rs index 7aa6a17c..f38bee27 100644 --- a/crates/spark/src/function/datetime/monthname.rs +++ b/crates/spark/src/function/datetime/monthname.rs @@ -2,10 +2,8 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - use arrow::compute::DatePart; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -15,6 +13,7 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::{date_part_name, date_signature}; +use crate::function::nullability::{SparkNullability, spark_nullability}; const MONTH_NAMES: [&str; 12] = [ "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", @@ -24,6 +23,9 @@ fn month_number_to_name(month: i32) -> Option<&'static str> { MONTH_NAMES.get((month - 1) as usize).copied() } +/// Spark's `MonthName` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `monthname` expression (Spark 4.0+). /// /// @@ -85,10 +87,11 @@ impl ScalarUDFImpl for SparkMonthName { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/datetime/next_day.rs b/crates/spark/src/function/datetime/next_day.rs index f386e86a..4eb078c5 100644 --- a/crates/spark/src/function/datetime/next_day.rs +++ b/crates/spark/src/function/datetime/next_day.rs @@ -4,10 +4,11 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, ArrayRef, AsArray, Date32Array, Date32Builder, StringArrayType, }; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef}; +use arrow::datatypes::{DataType, Date32Type, FieldRef}; use chrono::{Datelike, Weekday}; use datafusion_common::types::{NativeType, logical_date, logical_string}; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; @@ -17,6 +18,10 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `NextDay` is null-intolerant and always nullable, because an unknown day of the +/// week returns NULL when ANSI mode is off. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `next_day` expression. /// /// @@ -100,12 +105,12 @@ impl ScalarUDFImpl for SparkNextDay { internal_err!("return_field_from_args should be used instead") } - fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { - // Spark marks next_day as always nullable because invalid day_of_week values - // can yield NULL even when inputs are non-null. - Ok(Arc::new(Field::new(self.name(), DataType::Date32, true))) + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -373,6 +378,7 @@ fn spark_next_day( #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; use datafusion_common::config::ConfigOptions; #[test] diff --git a/crates/spark/src/function/datetime/to_utc_timestamp.rs b/crates/spark/src/function/datetime/to_utc_timestamp.rs index 674accbe..1790a75f 100644 --- a/crates/spark/src/function/datetime/to_utc_timestamp.rs +++ b/crates/spark/src/function/datetime/to_utc_timestamp.rs @@ -4,11 +4,12 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::timezone::Tz; use arrow::array::{Array, ArrayRef, AsArray, PrimitiveBuilder, StringArrayType}; use arrow::datatypes::TimeUnit; use arrow::datatypes::{ - ArrowTimestampType, DataType, Field, FieldRef, TimestampMicrosecondType, + ArrowTimestampType, DataType, FieldRef, TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; use chrono::{DateTime, Offset, TimeZone}; @@ -24,6 +25,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `ToUTCTimestamp` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `to_utc_timestamp` expression. /// /// @@ -102,15 +106,12 @@ impl ScalarUDFImpl for SparkToUtcTimestamp { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - - Ok(Arc::new(Field::new( - self.name(), - args.arg_fields[0].data_type().clone(), - nullable, - ))) + let data_type = args.arg_fields[0].data_type().clone(); + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(to_utc_timestamp, vec![])(&args.args) } diff --git a/crates/spark/src/function/datetime/unix.rs b/crates/spark/src/function/datetime/unix.rs index 2a6834d1..c0700ed3 100644 --- a/crates/spark/src/function/datetime/unix.rs +++ b/crates/spark/src/function/datetime/unix.rs @@ -4,7 +4,8 @@ use std::sync::{Arc, LazyLock}; -use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, TimeUnit}; +use crate::function::nullability::SparkNullability; +use arrow::datatypes::{DataType, FieldRef, Int64Type, TimeUnit}; use datafusion_common::cast::as_int64_array; use datafusion_common::types::logical_date; use datafusion_common::utils::take_function_args; @@ -17,6 +18,10 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `UnixDate`, `UnixSeconds`, `UnixMillis` and `UnixMicros` are null-intolerant +/// `UnaryExpression`s. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `unix_date` expression. /// /// @@ -74,8 +79,7 @@ impl ScalarUDFImpl for SparkUnixDate { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields[0].is_nullable(); - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } fn documentation(&self) -> Option<&Documentation> { @@ -173,8 +177,7 @@ impl ScalarUDFImpl for SparkUnixTimestamp { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields[0].is_nullable(); - Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int64, args.arg_fields)) } fn documentation(&self) -> Option<&Documentation> { @@ -264,7 +267,7 @@ NULL .downscale_divisor(&ts.get_type(info.schema())?) .is_some() { - return Ok(ExprSimplifyResult::Original(vec![ts])); + return NULLABILITY.simplify(self, vec![ts], info); } Ok(ExprSimplifyResult::Simplified( ts.cast_to( diff --git a/crates/spark/src/function/datetime/weekday.rs b/crates/spark/src/function/datetime/weekday.rs index fe2aa68c..74ead9ef 100644 --- a/crates/spark/src/function/datetime/weekday.rs +++ b/crates/spark/src/function/datetime/weekday.rs @@ -2,11 +2,10 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::AsArray; use arrow::compute::{DatePart, date_part}; -use arrow::datatypes::{DataType, Field, FieldRef, Int32Type}; +use arrow::datatypes::{DataType, FieldRef, Int32Type}; use datafusion_common::types::{NativeType, logical_date}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; @@ -16,6 +15,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `WeekDay` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `weekday` expression. /// /// @@ -79,10 +81,11 @@ impl ScalarUDFImpl for SparkWeekDay { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -113,6 +116,8 @@ impl ScalarUDFImpl for SparkWeekDay { mod tests { use super::*; use arrow::array::{Date32Array, Int32Array}; + use arrow::datatypes::Field; + use std::sync::Arc; #[test] fn test_weekday_return_field_nullability_matches_input() { diff --git a/crates/spark/src/function/functions_nested_utils.rs b/crates/spark/src/function/functions_nested_utils.rs index a6188de2..e8ef96ba 100644 --- a/crates/spark/src/function/functions_nested_utils.rs +++ b/crates/spark/src/function/functions_nested_utils.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use arrow::array::{Array, ArrayRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::ColumnarValue; @@ -38,3 +39,13 @@ where } } } + +/// The element field of a list type. +pub(crate) fn list_element_field(data_type: &DataType) -> Option<&FieldRef> { + match data_type { + DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { + Some(f) + } + _ => None, + } +} diff --git a/crates/spark/src/function/hash/crc32.rs b/crates/spark/src/function/hash/crc32.rs index 9bc224e9..8cd86385 100644 --- a/crates/spark/src/function/hash/crc32.rs +++ b/crates/spark/src/function/hash/crc32.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use arrow::array::{ArrayRef, Int64Array}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use crc32fast::Hasher; use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_fixed_size_binary_array, @@ -23,6 +23,11 @@ use datafusion_macros::user_doc; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `Crc32` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `crc32` expression. /// /// @@ -91,10 +96,11 @@ impl ScalarUDFImpl for SparkCrc32 { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int64, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -147,6 +153,7 @@ fn spark_crc32(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_crc32_nullability() -> Result<()> { diff --git a/crates/spark/src/function/hash/sha1.rs b/crates/spark/src/function/hash/sha1.rs index 650baf02..8164e712 100644 --- a/crates/spark/src/function/hash/sha1.rs +++ b/crates/spark/src/function/hash/sha1.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use arrow::array::{ArrayRef, StringArray, StringBuilder}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_fixed_size_binary_array, as_large_binary_array, @@ -24,6 +24,11 @@ use sha1::{Digest, Sha1}; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `Sha1` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `sha1` expression. /// /// @@ -99,10 +104,11 @@ impl ScalarUDFImpl for SparkSha1 { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -163,6 +169,7 @@ fn spark_sha1(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_sha1_nullability() -> Result<()> { diff --git a/crates/spark/src/function/hash/sha2.rs b/crates/spark/src/function/hash/sha2.rs index 8dc122cd..6721b2f6 100644 --- a/crates/spark/src/function/hash/sha2.rs +++ b/crates/spark/src/function/hash/sha2.rs @@ -20,6 +20,11 @@ use sha2::{self, Digest}; use std::sync::Arc; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `Sha2` is null-intolerant and always nullable, because an unsupported bit length +/// returns NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); /// Spark-compatible `sha2` expression. /// @@ -90,6 +95,8 @@ impl ScalarUDFImpl for SparkSha2 { Ok(DataType::Utf8) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/map/map_from_arrays.rs b/crates/spark/src/function/map/map_from_arrays.rs index 971f77f8..98def462 100644 --- a/crates/spark/src/function/map/map_from_arrays.rs +++ b/crates/spark/src/function/map/map_from_arrays.rs @@ -7,9 +7,10 @@ use crate::function::map::utils::{ get_element_type, get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, map_type_from_key_value_types, }; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, NullArray}; use arrow::compute::kernels::cast; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -17,7 +18,9 @@ use datafusion_expr::{ Volatility, }; use datafusion_functions::utils::make_scalar_function; -use std::sync::Arc; + +/// Spark's `MapFromArrays` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `map_from_arrays` expression /// @@ -62,12 +65,11 @@ impl ScalarUDFImpl for MapFromArrays { get_element_type(keys_field.data_type())?, get_element_type(values_field.data_type())?, ); - // Spark marks map_from_arrays as null intolerant, so the output is - // nullable if either input is nullable. - let nullable = keys_field.is_nullable() || values_field.is_nullable(); - Ok(Arc::new(Field::new(self.name(), map_type, nullable))) + Ok(NULLABILITY.return_field(self.name(), map_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let last_value_wins = map_last_value_wins(&args.config_options); make_scalar_function( @@ -104,6 +106,8 @@ fn map_from_arrays_inner(args: &[ArrayRef], last_value_wins: bool) -> Result #[derive(Debug, PartialEq, Eq, Hash)] @@ -83,11 +88,14 @@ impl ScalarUDFImpl for MapFromEntries { }?; let map_type = map_type_from_key_value_types(keys_type, values_type); - let nullable = entries_field.is_nullable() || entries_element_field.is_nullable(); + let nullable = + NULLABILITY.nullable(args.arg_fields) || entries_element_field.is_nullable(); Ok(Arc::new(Field::new(self.name(), map_type, nullable))) } + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let last_value_wins = map_last_value_wins(&args.config_options); make_scalar_function( diff --git a/crates/spark/src/function/map/str_to_map.rs b/crates/spark/src/function/map/str_to_map.rs index 039041c4..dd32858b 100644 --- a/crates/spark/src/function/map/str_to_map.rs +++ b/crates/spark/src/function/map/str_to_map.rs @@ -9,7 +9,7 @@ use arrow::array::{ Array, ArrayRef, MapBuilder, MapFieldNames, StringArrayType, StringBuilder, }; use arrow::buffer::NullBuffer; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::{ as_large_string_array, as_string_array, as_string_view_array, }; @@ -21,10 +21,14 @@ use datafusion_expr::{ use crate::config::map_last_value_wins; use crate::function::map::utils::map_type_from_key_value_types; +use crate::function::nullability::{SparkNullability, spark_nullability}; const DEFAULT_PAIR_DELIM: &str = ","; const DEFAULT_KV_DELIM: &str = ":"; +/// Spark's `StringToMap` is a null-intolerant `TernaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `str_to_map` expression /// /// @@ -83,11 +87,12 @@ impl ScalarUDFImpl for SparkStrToMap { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); let map_type = map_type_from_key_value_types(&DataType::Utf8, &DataType::Utf8); - Ok(Arc::new(Field::new(self.name(), map_type, nullable))) + Ok(NULLABILITY.return_field(self.name(), map_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let last_value_wins = map_last_value_wins(&args.config_options); let arrays: Vec = ColumnarValue::values_to_arrays(&args.args)?; diff --git a/crates/spark/src/function/math/abs.rs b/crates/spark/src/function/math/abs.rs index 6d81b9eb..879a8779 100644 --- a/crates/spark/src/function/math/abs.rs +++ b/crates/spark/src/function/math/abs.rs @@ -3,7 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use arrow::array::*; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use arrow::error::ArrowError; use datafusion_common::{DataFusionError, Result, ScalarValue, internal_err}; use datafusion_expr::{ @@ -17,11 +17,15 @@ use datafusion_macros::user_doc; use std::sync::Arc; use crate::function::error_utils::ARITHMETIC_OVERFLOW_ERROR; +use crate::function::nullability::{SparkNullability, spark_nullability}; fn arithmetic_overflow_error() -> ArrowError { ArrowError::ComputeError(ARITHMETIC_OVERFLOW_ERROR.to_string()) } +/// Spark's `Abs` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `abs` expression. /// /// @@ -82,13 +86,12 @@ impl ScalarUDFImpl for SparkAbs { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let input_field = &args.arg_fields[0]; - let out_dt = input_field.data_type().clone(); - let out_nullable = input_field.is_nullable(); - - Ok(Arc::new(Field::new(self.name(), out_dt, out_nullable))) + let data_type = args.arg_fields[0].data_type().clone(); + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -239,6 +242,7 @@ pub fn spark_abs( #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; use arrow::datatypes::i256; macro_rules! eval_array_legacy_mode { diff --git a/crates/spark/src/function/math/atan2.rs b/crates/spark/src/function/math/atan2.rs index 38227ee8..4d6647be 100644 --- a/crates/spark/src/function/math/atan2.rs +++ b/crates/spark/src/function/math/atan2.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray, Float64Array}; use arrow::compute::kernels::arity::binary; use arrow::datatypes::{DataType, Float64Type}; @@ -16,6 +17,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `Atan2` is a null-intolerant `BinaryMathExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `atan2` expression. /// /// @@ -80,6 +84,8 @@ impl ScalarUDFImpl for SparkAtan2 { Ok(DataType::Float64) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/bin.rs b/crates/spark/src/function/math/bin.rs index 5c2962aa..fa73503e 100644 --- a/crates/spark/src/function/math/bin.rs +++ b/crates/spark/src/function/math/bin.rs @@ -2,8 +2,9 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, AsArray, StringBuilder}; -use arrow::datatypes::{DataType, Field, FieldRef, Int64Type}; +use arrow::datatypes::{DataType, FieldRef, Int64Type}; use datafusion_common::types::{NativeType, logical_int64}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; @@ -15,6 +16,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Bin` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `bin` expression. /// /// @@ -81,13 +85,11 @@ impl ScalarUDFImpl for SparkBin { &self, args: datafusion_expr::ReturnFieldArgs, ) -> Result { - Ok(Arc::new(Field::new( - self.name(), - DataType::Utf8, - args.arg_fields[0].is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/ceil.rs b/crates/spark/src/function/math/ceil.rs index 6d08e5eb..ad482b66 100644 --- a/crates/spark/src/function/math/ceil.rs +++ b/crates/spark/src/function/math/ceil.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrowNativeTypeOp, AsArray, Decimal128Array}; use arrow::datatypes::{DataType, Decimal128Type, Float32Type, Float64Type, Int64Type}; use datafusion_common::utils::take_function_args; @@ -14,6 +15,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `Ceil` is a null-intolerant `UnaryMathExpression`, which is always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `ceil` expression. /// /// @@ -90,6 +94,8 @@ impl ScalarUDFImpl for SparkCeil { } } + spark_nullability!(NULLABILITY, return_type); + fn aliases(&self) -> &[String] { &self.aliases } diff --git a/crates/spark/src/function/math/divide.rs b/crates/spark/src/function/math/divide.rs index 4b9153dc..cada4e5e 100644 --- a/crates/spark/src/function/math/divide.rs +++ b/crates/spark/src/function/math/divide.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, Scalar, new_null_array}; use arrow::compute::kernels::boolean::{and, is_not_null}; use arrow::compute::kernels::numeric::div; @@ -54,6 +55,10 @@ fn spark_divide(args: &[ColumnarValue], enable_ansi_mode: bool) -> Result Result> { assert_eq_or_internal_err!( arg_types.len(), diff --git a/crates/spark/src/function/math/expm1.rs b/crates/spark/src/function/math/expm1.rs index b0c280df..e03e2150 100644 --- a/crates/spark/src/function/math/expm1.rs +++ b/crates/spark/src/function/math/expm1.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::error_utils::unsupported_data_type_exec_err; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray}; use arrow::datatypes::{DataType, Float64Type}; use datafusion_common::utils::take_function_args; @@ -14,6 +15,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Expm1` is a null-intolerant `UnaryMathExpression`, which is always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `expm1` expression. /// /// @@ -66,6 +70,8 @@ impl ScalarUDFImpl for SparkExpm1 { Ok(DataType::Float64) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/factorial.rs b/crates/spark/src/function/math/factorial.rs index 7070e0f5..ecaac754 100644 --- a/crates/spark/src/function/math/factorial.rs +++ b/crates/spark/src/function/math/factorial.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, Int64Array}; use arrow::datatypes::DataType; use arrow::datatypes::DataType::{Int32, Int64}; @@ -19,6 +20,10 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `Factorial` is null-intolerant and always nullable, because a value outside 0 to +/// 20 returns NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `factorial` expression. /// /// @@ -81,6 +86,8 @@ impl ScalarUDFImpl for SparkFactorial { Ok(Int64) } + spark_nullability!(NULLABILITY, return_type); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_factorial(&args.args) } diff --git a/crates/spark/src/function/math/floor.rs b/crates/spark/src/function/math/floor.rs index 44b4ab43..18673ca3 100644 --- a/crates/spark/src/function/math/floor.rs +++ b/crates/spark/src/function/math/floor.rs @@ -2,11 +2,12 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::cast::AsArray; use arrow::array::types::Decimal128Type; use arrow::array::{ArrowNativeTypeOp, Decimal128Array, Int64Array}; use arrow::compute::kernels::arity::unary; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{DataFusionError, ScalarValue, exec_err, internal_err}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, @@ -15,6 +16,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Floor` is a null-intolerant `UnaryMathExpression`, which is always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `floor` expression. /// /// @@ -79,7 +83,6 @@ impl ScalarUDFImpl for SparkFloor { &self, args: ReturnFieldArgs, ) -> datafusion_common::Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); let return_type = match args.arg_fields[0].data_type() { DataType::Decimal128(p, s) if *s > 0 => { let new_p = (*p - *s as u8 + 1).clamp(1, 38); @@ -97,9 +100,11 @@ impl ScalarUDFImpl for SparkFloor { args.arg_fields[0].data_type() )?, }; - Ok(Arc::new(Field::new(self.name(), return_type, nullable))) + Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/hex.rs b/crates/spark/src/function/math/hex.rs index 07dacdfa..7bdc3aec 100644 --- a/crates/spark/src/function/math/hex.rs +++ b/crates/spark/src/function/math/hex.rs @@ -5,6 +5,7 @@ use std::str::from_utf8_unchecked; use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayAccessor, ArrayRef, StringArray, StringBuilder}; use arrow::buffer::{Buffer, OffsetBuffer}; use arrow::datatypes::DataType; @@ -28,6 +29,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `Hex` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `hex` expression. /// /// @@ -110,6 +114,8 @@ impl ScalarUDFImpl for SparkHex { }) } + spark_nullability!(NULLABILITY, return_type); + fn invoke_with_args( &self, args: ScalarFunctionArgs, diff --git a/crates/spark/src/function/math/hypot.rs b/crates/spark/src/function/math/hypot.rs index 52f26248..480e766b 100644 --- a/crates/spark/src/function/math/hypot.rs +++ b/crates/spark/src/function/math/hypot.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray, Float64Array}; use arrow::compute::kernels::arity::binary; use arrow::datatypes::{DataType, Float64Type}; @@ -16,6 +17,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `Hypot` is a null-intolerant `BinaryMathExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `hypot` expression. /// /// @@ -79,6 +83,8 @@ impl ScalarUDFImpl for SparkHypot { Ok(DataType::Float64) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/modulus.rs b/crates/spark/src/function/math/modulus.rs index 31c49a40..6e01773f 100644 --- a/crates/spark/src/function/math/modulus.rs +++ b/crates/spark/src/function/math/modulus.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, BooleanArray, Scalar, new_null_array}; use arrow::compute::kernels::cast::{CastOptions, cast, cast_with_options}; use arrow::compute::kernels::numeric::add_wrapping; @@ -192,6 +193,10 @@ pub fn spark_pmod( Ok(ColumnarValue::Array(result)) } +/// Spark's `Remainder` and `Pmod` are null-intolerant and always nullable, because a zero +/// divisor returns NULL when ANSI mode is off. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// SparkMod implements the Spark-compatible modulo function #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkMod { @@ -233,6 +238,8 @@ impl ScalarUDFImpl for SparkMod { Ok(arg_types[0].clone()) } + spark_nullability!(NULLABILITY, return_type); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_mod(&args.args, args.config_options.execution.enable_ansi_mode) } @@ -298,6 +305,8 @@ impl ScalarUDFImpl for SparkPmod { } } + spark_nullability!(NULLABILITY, return_type); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_pmod( &args.args, diff --git a/crates/spark/src/function/math/negative.rs b/crates/spark/src/function/math/negative.rs index 3ad5c401..1d55faa6 100644 --- a/crates/spark/src/function/math/negative.rs +++ b/crates/spark/src/function/math/negative.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::types::*; use arrow::array::*; use arrow::datatypes::{DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit}; @@ -15,6 +16,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `UnaryMinus` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `negative` expression. /// /// @@ -87,6 +91,8 @@ impl ScalarUDFImpl for SparkNegative { Ok(arg_types[0].clone()) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/pow.rs b/crates/spark/src/function/math/pow.rs index 7716ae01..7a467d26 100644 --- a/crates/spark/src/function/math/pow.rs +++ b/crates/spark/src/function/math/pow.rs @@ -14,6 +14,7 @@ use arrow::array::{Array, ArrayRef, Float64Array}; use arrow::compute::binary; use arrow::datatypes::DataType; +use crate::function::nullability::{SparkNullability, spark_nullability}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::{ @@ -21,6 +22,9 @@ use datafusion_expr::{ }; use datafusion_functions::math::power::PowerFunc; +/// Spark's `Pow` is a null-intolerant `BinaryMathExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible implementation of `pow` / `power`. /// /// Behavioural differences from the DataFusion default: @@ -70,6 +74,8 @@ impl ScalarUDFImpl for SparkPow { self.inner.return_type(arg_types) } + spark_nullability!(NULLABILITY, return_type); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { // Only Float64 × Float64 needs the Spark override. // Decimal / integer / mixed-type paths are delegated to the standard diff --git a/crates/spark/src/function/math/rint.rs b/crates/spark/src/function/math/rint.rs index c925dd2c..2174790d 100644 --- a/crates/spark/src/function/math/rint.rs +++ b/crates/spark/src/function/math/rint.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, AsArray}; use arrow::compute::cast; use arrow::datatypes::DataType::{ @@ -19,6 +20,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `Rint` is a null-intolerant `UnaryMathExpression`, which is always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `rint` expression. /// /// @@ -71,6 +75,8 @@ impl ScalarUDFImpl for SparkRint { Ok(Float64) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/unhex.rs b/crates/spark/src/function/math/unhex.rs index 41aa6403..e23c356e 100644 --- a/crates/spark/src/function/math/unhex.rs +++ b/crates/spark/src/function/math/unhex.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, BinaryBuilder}; use arrow::datatypes::DataType; use datafusion_common::cast::{ @@ -19,6 +20,10 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Unhex` is null-intolerant and always nullable, because an invalid hex string +/// returns NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `unhex` expression. /// /// @@ -77,6 +82,8 @@ impl ScalarUDFImpl for SparkUnhex { Ok(DataType::Binary) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/width_bucket.rs b/crates/spark/src/function/math/width_bucket.rs index 4e2ba485..f2391d5c 100644 --- a/crates/spark/src/function/math/width_bucket.rs +++ b/crates/spark/src/function/math/width_bucket.rs @@ -27,11 +27,16 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Int64Array, Int64Builder}; use arrow::datatypes::TimeUnit::Microsecond; use datafusion_expr::Coercion; use datafusion_expr::Volatility::Immutable; +/// Spark's `WidthBucket` is null-intolerant and always nullable, because invalid bounds or +/// bucket counts return NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `width_bucket` expression. /// /// @@ -155,6 +160,8 @@ impl ScalarUDFImpl for SparkWidthBucket { Ok(Int64) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 1c165bed..a2fc12b2 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -23,6 +23,7 @@ pub mod map; pub mod math; pub mod misc; mod null_utils; +pub(crate) mod nullability; pub mod predicate; pub mod string; pub mod r#struct; diff --git a/crates/spark/src/function/nullability.rs b/crates/spark/src/function/nullability.rs new file mode 100644 index 00000000..07fd7ad9 --- /dev/null +++ b/crates/spark/src/function/nullability.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! A function's nullability, declared as Spark declares it. +//! +//! Spark describes it with two members of `Expression`: +//! +//! - `nullable`: whether the result can be NULL. `UnaryExpression`, `BinaryExpression` and +//! `TernaryExpression` take it from their children (`children.exists(_.nullable)`), and an +//! expression that can return NULL for non-NULL inputs overrides it, as `next_day` does. +//! - `nullIntolerant` (the `NullIntolerant` trait before Spark 4.0): a NULL input always gives a +//! NULL output. The `NullPropagation` optimizer rule uses it to replace a call that has a NULL +//! literal argument with a NULL literal. +//! +//! These are independent: a null-intolerant expression can still be nullable for non-NULL +//! inputs. +//! +//! Every function that uses [`SparkNullability`] is null-intolerant, so it declares only the +//! `nullable` rule, and [`spark_nullability!`] implements `simplify` from it: +//! +//! ```ignore +//! const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); +//! +//! impl ScalarUDFImpl for SparkAscii { +//! fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { +//! Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) +//! } +//! +//! spark_nullability!(NULLABILITY); +//! } +//! ``` +//! +//! A function that computes its type in `return_type` uses +//! `spark_nullability!(NULLABILITY, return_type)`, which also implements +//! `return_field_from_args`. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::{Result, ScalarValue}; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use datafusion_expr::{Expr, ExprSchemable, ReturnFieldArgs, ScalarUDFImpl, lit}; + +/// The nullability of a null-intolerant function: when its result can be NULL, as Spark's +/// `Expression.nullable` declares it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum SparkNullability { + /// Nullable when any argument is: `children.exists(_.nullable)`, the default of + /// `UnaryExpression`, `BinaryExpression` and `TernaryExpression`. + Children, + /// Always nullable, because the function can return NULL for non-NULL arguments. + Always, +} + +impl SparkNullability { + /// A null-intolerant function whose result is nullable when an argument is. This is a + /// `UnaryExpression`, `BinaryExpression` or `TernaryExpression` with + /// `nullIntolerant = true` that does not override `nullable`. + pub(crate) const fn null_intolerant() -> Self { + Self::Children + } + + /// A null-intolerant function that can return NULL for non-NULL arguments, so its result + /// is always nullable. Spark overrides `nullable` to `true` for these. + pub(crate) const fn null_intolerant_always_nullable() -> Self { + Self::Always + } + + /// Whether the result can be NULL, given the argument fields. + pub(crate) fn nullable(&self, arg_fields: &[FieldRef]) -> bool { + match self { + Self::Children => arg_fields.iter().any(|f| f.is_nullable()), + Self::Always => true, + } + } + + /// The return field, for `ScalarUDFImpl::return_field_from_args`. + pub(crate) fn return_field( + &self, + name: &str, + data_type: DataType, + arg_fields: &[FieldRef], + ) -> FieldRef { + Arc::new(Field::new(name, data_type, self.nullable(arg_fields))) + } + + /// The return field of a function that computes its type in `ScalarUDFImpl::return_type`, + /// for `ScalarUDFImpl::return_field_from_args`. + pub(crate) fn return_field_from_type( + &self, + func: &dyn ScalarUDFImpl, + args: &ReturnFieldArgs, + ) -> Result { + let arg_types = args + .arg_fields + .iter() + .map(|f| f.data_type().clone()) + .collect::>(); + let data_type = func.return_type(&arg_types)?; + Ok(self.return_field(func.name(), data_type, args.arg_fields)) + } + + /// For `ScalarUDFImpl::simplify`: replaces a call that has a NULL literal argument with a + /// NULL literal of its return type, as Spark's `NullPropagation` does for a + /// null-intolerant expression. Otherwise returns the arguments unchanged. + /// + /// This is only an optimization: `invoke_with_args` must return NULL for such a call too. + pub(crate) fn simplify( + &self, + func: &dyn ScalarUDFImpl, + args: Vec, + info: &SimplifyContext, + ) -> Result { + if !args.iter().any(is_null_literal) { + return Ok(ExprSimplifyResult::Original(args)); + } + let arg_fields = args + .iter() + .map(|arg| Ok(arg.to_field(info.schema())?.1)) + .collect::>>()?; + let scalar_arguments = args + .iter() + .map(|arg| match arg { + Expr::Literal(value, _) => Some(value), + _ => None, + }) + .collect::>(); + let return_field = func.return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &scalar_arguments, + })?; + match ScalarValue::try_new_null(return_field.data_type()) { + Ok(null) => Ok(ExprSimplifyResult::Simplified(lit(null))), + // A type that has no NULL `ScalarValue` is left to `invoke_with_args` + Err(_) => Ok(ExprSimplifyResult::Original(args)), + } + } +} + +/// Implements `ScalarUDFImpl` methods from a function's [`SparkNullability`], inside its +/// `impl ScalarUDFImpl` block: +/// +/// - `spark_nullability!(NULLABILITY)` implements `simplify` with +/// [`SparkNullability::simplify`]. +/// - `spark_nullability!(NULLABILITY, return_type)` also implements `return_field_from_args` +/// with [`SparkNullability::return_field_from_type`], for a function that computes its type +/// in `return_type`. +macro_rules! spark_nullability { + ($nullability:expr) => { + fn simplify( + &self, + args: Vec, + info: &datafusion_expr::simplify::SimplifyContext, + ) -> datafusion_common::Result { + $nullability.simplify(self, args, info) + } + }; + ($nullability:expr, return_type) => { + $crate::function::nullability::spark_nullability!($nullability); + + fn return_field_from_args( + &self, + args: datafusion_expr::ReturnFieldArgs, + ) -> datafusion_common::Result { + $nullability.return_field_from_type(self, &args) + } + }; +} +pub(crate) use spark_nullability; + +fn is_null_literal(expr: &Expr) -> bool { + matches!(expr, Expr::Literal(value, _) if value.is_null()) +} + +#[cfg(test)] +mod tests { + use super::*; + + use datafusion_common::{DFSchema, ScalarValue}; + use datafusion_expr::col; + + use crate::function::string::ascii::SparkAscii; + + fn fields(nullable: &[bool]) -> Vec { + nullable + .iter() + .enumerate() + .map(|(i, n)| Arc::new(Field::new(format!("a{i}"), DataType::Utf8, *n))) + .collect() + } + + #[test] + fn test_nullable_from_children() { + let nullability = SparkNullability::null_intolerant(); + assert!(!nullability.nullable(&[])); + assert!(!nullability.nullable(&fields(&[false, false]))); + assert!(nullability.nullable(&fields(&[false, true]))); + } + + #[test] + fn test_nullable_always() { + let always = SparkNullability::null_intolerant_always_nullable(); + assert!(always.nullable(&[])); + assert!(always.nullable(&fields(&[false]))); + } + + #[test] + fn test_return_field() { + let field = SparkNullability::null_intolerant().return_field( + "f", + DataType::Int32, + &fields(&[true]), + ); + assert_eq!(field.as_ref(), &Field::new("f", DataType::Int32, true)); + } + + fn schema() -> SimplifyContext { + let schema = DFSchema::try_from(arrow::datatypes::Schema::new(vec![Field::new( + "s", + DataType::Utf8, + true, + )])) + .unwrap(); + SimplifyContext::builder() + .with_schema(Arc::new(schema)) + .build() + } + + #[test] + fn test_simplify_null_literal_to_typed_null() { + let result = SparkNullability::null_intolerant() + .simplify( + &SparkAscii::new(), + vec![lit(ScalarValue::Utf8(None))], + &schema(), + ) + .unwrap(); + let ExprSimplifyResult::Simplified(expr) = result else { + panic!("expected a NULL literal"); + }; + assert_eq!(expr, lit(ScalarValue::Int32(None))); + } + + #[test] + fn test_simplify_keeps_non_null_arguments() { + for args in [vec![col("s")], vec![lit("a")]] { + let result = SparkNullability::null_intolerant() + .simplify(&SparkAscii::new(), args.clone(), &schema()) + .unwrap(); + let ExprSimplifyResult::Original(original) = result else { + panic!("expected the arguments unchanged"); + }; + assert_eq!(original, args); + } + } +} diff --git a/crates/spark/src/function/string/ascii.rs b/crates/spark/src/function/string/ascii.rs index 35e1af7a..eb8670a2 100644 --- a/crates/spark/src/function/string/ascii.rs +++ b/crates/spark/src/function/string/ascii.rs @@ -2,9 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::types::{NativeType, logical_string}; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -15,6 +13,11 @@ use datafusion_functions::string::ascii::ascii; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `Ascii` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `ascii` expression. /// /// @@ -81,12 +84,11 @@ impl ScalarUDFImpl for SparkAscii { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - // ascii returns an Int32 value - // The result is nullable only if any of the input arguments is nullable - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new("ascii", DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -99,6 +101,8 @@ impl ScalarUDFImpl for SparkAscii { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; + use std::sync::Arc; #[test] fn test_return_field_nullable_input() { diff --git a/crates/spark/src/function/string/base64.rs b/crates/spark/src/function/string/base64.rs index 794eadef..55cc832c 100644 --- a/crates/spark/src/function/string/base64.rs +++ b/crates/spark/src/function/string/base64.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, GenericBinaryBuilder, OffsetSizeTrait}; use arrow::datatypes::DataType; use datafusion_common::arrow::datatypes::{Field, FieldRef}; @@ -119,6 +120,9 @@ impl ScalarUDFImpl for SparkBase64 { } } +/// Spark's `UnBase64` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `unbase64` expression. /// /// @@ -191,13 +195,11 @@ impl ScalarUDFImpl for SparkUnBase64 { DataType::LargeBinary => DataType::LargeBinary, _ => DataType::Binary, }; - Ok(Arc::new(Field::new( - self.name(), - return_type, - str.is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_unbase64, vec![])(&args.args) } diff --git a/crates/spark/src/function/string/char.rs b/crates/spark/src/function/string/char.rs index bf378607..6f8bf66a 100644 --- a/crates/spark/src/function/string/char.rs +++ b/crates/spark/src/function/string/char.rs @@ -6,7 +6,7 @@ use arrow::array::ArrayRef; use arrow::array::GenericStringBuilder; use arrow::datatypes::DataType::Int64; use arrow::datatypes::DataType::Utf8; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_macros::user_doc; use std::sync::Arc; @@ -16,6 +16,11 @@ use datafusion_expr::{ Signature, Volatility, }; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `Chr` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `char` expression. /// /// @@ -81,9 +86,10 @@ impl ScalarUDFImpl for CharFunc { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), Utf8, nullable))) + Ok(NULLABILITY.return_field(self.name(), Utf8, args.arg_fields)) } + + spark_nullability!(NULLABILITY); } /// Returns the ASCII character having the binary equivalent to the input expression. diff --git a/crates/spark/src/function/string/contains.rs b/crates/spark/src/function/string/contains.rs index 19bed03c..266d367d 100644 --- a/crates/spark/src/function/string/contains.rs +++ b/crates/spark/src/function/string/contains.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, BooleanArray, Scalar}; use arrow::compute::kernels::cast::cast; use arrow::compute::kernels::comparison::contains as arrow_contains; @@ -16,6 +17,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `Contains` is a null-intolerant `StringPredicate`, a `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `contains` expression. /// /// @@ -156,6 +160,8 @@ impl ScalarUDFImpl for SparkContains { Ok(DataType::Boolean) } + spark_nullability!(NULLABILITY, return_type); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/string/is_valid_utf8.rs b/crates/spark/src/function/string/is_valid_utf8.rs index 7c93e9bb..6a4dd9c9 100644 --- a/crates/spark/src/function/string/is_valid_utf8.rs +++ b/crates/spark/src/function/string/is_valid_utf8.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, @@ -18,8 +18,12 @@ use datafusion_common::cast::{ use datafusion_common::utils::take_function_args; use datafusion_functions::utils::make_scalar_function; +use crate::function::nullability::{SparkNullability, spark_nullability}; use std::sync::Arc; +/// Spark's `IsValidUTF8` is null-intolerant and always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `is_valid_utf8` expression. /// /// @@ -86,10 +90,12 @@ impl ScalarUDFImpl for SparkIsValidUtf8 { internal_err!("return_field_from_args should be used instead") } - fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { - Ok(Arc::new(Field::new(self.name(), DataType::Boolean, true))) + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + Ok(NULLABILITY.return_field(self.name(), DataType::Boolean, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/string/length.rs b/crates/spark/src/function/string/length.rs index 632fbe91..e19248d6 100644 --- a/crates/spark/src/function/string/length.rs +++ b/crates/spark/src/function/string/length.rs @@ -5,7 +5,7 @@ use arrow::array::{ Array, ArrayRef, AsArray, BinaryArrayType, PrimitiveArray, StringArrayType, }; -use arrow::datatypes::{DataType, Field, FieldRef, Int32Type}; +use arrow::datatypes::{DataType, FieldRef, Int32Type}; use datafusion_common::exec_err; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, @@ -15,6 +15,11 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `Length` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `length` expression. /// /// @@ -109,10 +114,11 @@ impl ScalarUDFImpl for SparkLengthFunc { &self, args: ReturnFieldArgs, ) -> datafusion_common::Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); // spark length always returns Int32 - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } + + spark_nullability!(NULLABILITY); } fn spark_length(args: &[ArrayRef]) -> datafusion_common::Result { @@ -214,6 +220,7 @@ mod tests { use crate::function::utils::test::test_scalar_function; use arrow::array::Int32Array; use arrow::datatypes::DataType::Int32; + use arrow::datatypes::Field; use datafusion_common::{Result, ScalarValue}; macro_rules! test_spark_length_string { diff --git a/crates/spark/src/function/string/like.rs b/crates/spark/src/function/string/like.rs index 32218503..1e582af2 100644 --- a/crates/spark/src/function/string/like.rs +++ b/crates/spark/src/function/string/like.rs @@ -2,9 +2,10 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::ArrayRef; use arrow::compute::like; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::ColumnarValue; use datafusion_expr::{ @@ -15,6 +16,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Like` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `like` expression. /// /// @@ -80,14 +84,11 @@ impl ScalarUDFImpl for SparkLike { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new( - self.name(), - DataType::Boolean, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Boolean, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_like, vec![])(&args.args) } @@ -109,6 +110,7 @@ mod tests { use crate::function::utils::test::test_scalar_function; use arrow::array::{Array, BooleanArray}; use arrow::datatypes::DataType::Boolean; + use arrow::datatypes::Field; use datafusion_common::ScalarValue; macro_rules! test_like_string_invoke { diff --git a/crates/spark/src/function/string/make_valid_utf8.rs b/crates/spark/src/function/string/make_valid_utf8.rs index fbf75063..60ec52f8 100644 --- a/crates/spark/src/function/string/make_valid_utf8.rs +++ b/crates/spark/src/function/string/make_valid_utf8.rs @@ -2,8 +2,9 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, LargeStringArray, StringArray}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_large_binary_array, }; @@ -17,6 +18,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `MakeValidUTF8` is null-intolerant and always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `make_valid_utf8` expression. /// /// @@ -93,13 +97,11 @@ impl ScalarUDFImpl for SparkMakeValidUtf8 { DataType::LargeBinary => Ok(DataType::LargeUtf8), data_type => internal_err!("make_valid_utf8 does not support: {data_type}"), }?; - Ok(Arc::new(Field::new( - self.name(), - return_type, - make_valid_utf8.is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) } + spark_nullability!(NULLABILITY); + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/string/soundex.rs b/crates/spark/src/function/string/soundex.rs index 2b533e80..d62631f6 100644 --- a/crates/spark/src/function/string/soundex.rs +++ b/crates/spark/src/function/string/soundex.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait}; use arrow::datatypes::DataType; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; @@ -13,6 +14,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `SoundEx` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `soundex` expression. /// /// @@ -72,6 +76,8 @@ impl ScalarUDFImpl for SparkSoundex { } } + spark_nullability!(NULLABILITY, return_type); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_soundex_inner, vec![])(&args.args) } diff --git a/crates/spark/src/function/string/space.rs b/crates/spark/src/function/string/space.rs index dea5e05e..e89dab24 100644 --- a/crates/spark/src/function/string/space.rs +++ b/crates/spark/src/function/string/space.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, ArrayRef, DictionaryArray, Int32Array, StringArray, StringBuilder, as_dictionary_array, @@ -16,6 +17,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `StringSpace` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `space` expression. /// /// @@ -86,6 +90,8 @@ impl ScalarUDFImpl for SparkSpace { Ok(return_type) } + spark_nullability!(NULLABILITY, return_type); + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_space(&args.args) } diff --git a/crates/spark/src/function/string/substring.rs b/crates/spark/src/function/string/substring.rs index 3fda3918..3f0d542b 100644 --- a/crates/spark/src/function/string/substring.rs +++ b/crates/spark/src/function/string/substring.rs @@ -8,7 +8,7 @@ use arrow::array::{ StringViewBuilder, }; use arrow::datatypes::DataType; -use datafusion_common::arrow::datatypes::{Field, FieldRef}; +use datafusion_common::arrow::datatypes::FieldRef; use datafusion_common::cast::as_int64_array; use datafusion_common::types::{ NativeType, logical_int32, logical_int64, logical_string, @@ -22,7 +22,11 @@ use datafusion_expr::{ use datafusion_functions::unicode::substr::{enable_ascii_fast_path, get_true_start_end}; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; -use std::sync::Arc; + +use crate::function::nullability::{SparkNullability, spark_nullability}; + +/// Spark's `Substring` is a null-intolerant `TernaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `substring` expression /// @@ -142,15 +146,14 @@ impl ScalarUDFImpl for SparkSubstring { } fn return_field_from_args(&self, args: ReturnFieldArgs<'_>) -> Result { - // Spark semantics: substring returns NULL if ANY input is NULL - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - - Ok(Arc::new(Field::new( + Ok(NULLABILITY.return_field( "substring", args.arg_fields[0].data_type().clone(), - nullable, - ))) + args.arg_fields, + )) } + + spark_nullability!(NULLABILITY); } fn spark_substring(args: &[ArrayRef]) -> Result { diff --git a/crates/sqllogictest/test_files/array/slice.slt b/crates/sqllogictest/test_files/array/slice.slt index e7f50a0d..368b2389 100644 --- a/crates/sqllogictest/test_files/array/slice.slt +++ b/crates/sqllogictest/test_files/array/slice.slt @@ -145,3 +145,11 @@ query ?T SELECT slice(array(1, 2, 3, 4), 2, 2), arrow_typeof(slice(array(1, 2, 3, 4), 2, 2)); ---- [2, 3] List(Int64, field: 'element') + +# slice is null-intolerant, as in Spark: a NULL argument makes every row NULL +query ?? +SELECT slice(column1, NULL::int, 2::int), slice(column1, 1::int, NULL::int) +FROM VALUES (array(1, 2, 3)), (NULL); +---- +NULL NULL +NULL NULL diff --git a/crates/sqllogictest/test_files/bitwise/bit_get.slt b/crates/sqllogictest/test_files/bitwise/bit_get.slt index d93a827d..76f508a6 100644 --- a/crates/sqllogictest/test_files/bitwise/bit_get.slt +++ b/crates/sqllogictest/test_files/bitwise/bit_get.slt @@ -85,3 +85,11 @@ query I SELECT bit_get(11, 3::bigint); ---- 1 + +# bit_get is null-intolerant, as in Spark: a NULL argument makes every row NULL +query I +SELECT bit_get(column1, NULL::int) +FROM VALUES (11), (NULL); +---- +NULL +NULL diff --git a/crates/sqllogictest/test_files/datetime/date_add.slt b/crates/sqllogictest/test_files/datetime/date_add.slt index 385a4a23..82298e81 100644 --- a/crates/sqllogictest/test_files/datetime/date_add.slt +++ b/crates/sqllogictest/test_files/datetime/date_add.slt @@ -80,3 +80,11 @@ SELECT date_add('2024-01-01', 1::INT); # xfail spark: An integer literal is BIGINT here and INT in Spark (https://github.com/apache/datafusion-spark/issues/97) statement error Failed to coerce arguments to satisfy a call to 'date_add' function SELECT date_add(DATE '2024-01-01', 1); + +# date_add is null-intolerant, as in Spark: a NULL argument makes every row NULL +query D +SELECT date_add(column1, NULL::int) +FROM VALUES (DATE '2024-01-01'), (NULL); +---- +NULL +NULL diff --git a/crates/sqllogictest/test_files/datetime/date_sub.slt b/crates/sqllogictest/test_files/datetime/date_sub.slt index 11806234..8bcf7ae0 100644 --- a/crates/sqllogictest/test_files/datetime/date_sub.slt +++ b/crates/sqllogictest/test_files/datetime/date_sub.slt @@ -79,3 +79,11 @@ SELECT date_sub('2024-01-01', 1::INT); # xfail spark: An integer literal is BIGINT here and INT in Spark (https://github.com/apache/datafusion-spark/issues/97) statement error Failed to coerce arguments to satisfy a call to 'date_sub' function SELECT date_sub(DATE '2024-01-01', 1); + +# date_sub is null-intolerant, as in Spark: a NULL argument makes every row NULL +query D +SELECT date_sub(column1, NULL::int) +FROM VALUES (DATE '2024-01-01'), (NULL); +---- +NULL +NULL diff --git a/crates/sqllogictest/test_files/datetime/make_dt_interval.slt b/crates/sqllogictest/test_files/datetime/make_dt_interval.slt index be19b359..2a77ad0a 100644 --- a/crates/sqllogictest/test_files/datetime/make_dt_interval.slt +++ b/crates/sqllogictest/test_files/datetime/make_dt_interval.slt @@ -143,9 +143,6 @@ SELECT make_dt_interval(1491308, 0, 0, 0); ---- 1491308 days 0 hours 0 mins 0.000000 secs -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(1491309, 0, 0, 0); ---- @@ -167,9 +164,6 @@ SELECT make_dt_interval(106751991, 0, 0, 0); 106751991 days 0 hours 0 mins 0.000000 secs # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(106751992, 0, 0, 0); ---- @@ -181,9 +175,6 @@ SELECT make_dt_interval(-106751991, 0, 0, 0); -106751991 days 0 hours 0 mins 0.000000 secs # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(-106751992, 0, 0, 0); ---- @@ -191,18 +182,12 @@ NULL # Mixed-sign cancellation checks (Spark raises INTERVAL_ARITHMETIC_OVERFLOW, returns NULL pending #20) # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(106751992, -24, 0, 0); ---- NULL # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(106751991, 5, -60, 0); ---- @@ -221,18 +206,12 @@ SELECT make_dt_interval(-106751991, -4, 0, -54.775808); # One microsecond beyond boundaries returns NULL (overflow tracked in #20) # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(106751991, 4, 0, 54.775808); ---- NULL # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(-106751991, -4, 0, -54.775809); ---- diff --git a/crates/sqllogictest/test_files/math/atan2.slt b/crates/sqllogictest/test_files/math/atan2.slt index 4b837a5b..6ad33337 100644 --- a/crates/sqllogictest/test_files/math/atan2.slt +++ b/crates/sqllogictest/test_files/math/atan2.slt @@ -166,3 +166,11 @@ SELECT atan2(a, b) FROM (VALUES (0.0::double, 1.0::double), (1.0::double, 1.0::d 0.7853981633974483 NULL 3.141592653589793 + +# atan2 is null-intolerant, as in Spark: a NULL argument makes every row NULL +query RR +SELECT atan2(column1, NULL::double), atan2(NULL::double, column1) +FROM VALUES (1.0::double), (NULL::double); +---- +NULL NULL +NULL NULL diff --git a/crates/sqllogictest/test_files/math/pmod.slt b/crates/sqllogictest/test_files/math/pmod.slt index 7a7b4e63..9efcda1e 100644 --- a/crates/sqllogictest/test_files/math/pmod.slt +++ b/crates/sqllogictest/test_files/math/pmod.slt @@ -560,3 +560,11 @@ query R SELECT pmod(-7.2, 3.0) as pmod_scalar_4; ---- 1.7999999999999998 + +# pmod is null-intolerant, as in Spark: a NULL argument makes every row NULL +query II +SELECT pmod(column1, NULL::int), pmod(NULL::int, column1) +FROM VALUES (7::int), (NULL::int); +---- +NULL NULL +NULL NULL diff --git a/crates/sqllogictest/test_files/predicate/like.slt b/crates/sqllogictest/test_files/predicate/like.slt index 57f71e9a..a73c9781 100644 --- a/crates/sqllogictest/test_files/predicate/like.slt +++ b/crates/sqllogictest/test_files/predicate/like.slt @@ -69,3 +69,11 @@ query B SELECT like('Spark'::string, NULL::string); ---- NULL + +# like is null-intolerant, as in Spark: a NULL argument makes every row NULL +query B +SELECT like(column1, NULL::string) +FROM VALUES ('Spark SQL'::string), (NULL::string); +---- +NULL +NULL diff --git a/crates/sqllogictest/test_files/string/contains.slt b/crates/sqllogictest/test_files/string/contains.slt index ade7a622..02266ae9 100644 --- a/crates/sqllogictest/test_files/string/contains.slt +++ b/crates/sqllogictest/test_files/string/contains.slt @@ -175,3 +175,11 @@ true NULL NULL false + +# contains is null-intolerant, as in Spark: a NULL argument makes every row NULL +query BB +SELECT contains(column1, NULL::string), contains(NULL::string, column1) +FROM VALUES ('Spark SQL'::string), (NULL::string); +---- +NULL NULL +NULL NULL diff --git a/crates/sqllogictest/test_files/string/substring.slt b/crates/sqllogictest/test_files/string/substring.slt index 153575f6..58af8549 100644 --- a/crates/sqllogictest/test_files/string/substring.slt +++ b/crates/sqllogictest/test_files/string/substring.slt @@ -360,3 +360,24 @@ FROM VALUES NULL NULL NULL NULL NULL NULL + +# substring is null-intolerant, as in Spark: a NULL argument makes every row NULL +query TT +SELECT substring(column1, NULL::int), substring(column1, 2::int, NULL::int) +FROM VALUES ('Spark SQL'::string), (NULL::string); +---- +NULL NULL +NULL NULL + +# The simplifier replaces a call with a NULL literal argument with a NULL literal, as +# Spark's NullPropagation does. Without the simplifier the call is planned as written. +skipif evaluate +query TT +EXPLAIN SELECT substring(column1, NULL::int) FROM VALUES ('Spark SQL'::string); +---- +logical_plan +01)Projection: Utf8View(NULL) AS substring(column1,NULL) +02)--Values: (Utf8View("Spark SQL") AS Utf8("Spark SQL")) +physical_plan +01)ProjectionExec: expr=[NULL as substring(column1,NULL)] +02)--DataSourceExec: partitions=1, partition_sizes=[1]