From a1c6e0993368709c13fbfe45cbc7a9e011f7331d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 07:12:31 -0700 Subject: [PATCH 1/5] feat(spark): declare nullable and nullIntolerant with SparkNullability Add SparkNullability, which declares a function's nullability as Spark's nullable and nullIntolerant members do. It provides the return field and, for a null-intolerant function, a simplify that replaces a call with a NULL literal argument with a NULL literal, as Spark's NullPropagation does. Move ascii, char, length, crc32, sha1, date_add, date_sub, bit_get, substring and slice onto it. Part of #383. --- crates/spark/src/function/array/slice.rs | 19 +- crates/spark/src/function/bitwise/bit_get.rs | 22 +- .../spark/src/function/datetime/date_add.rs | 25 +- .../spark/src/function/datetime/date_sub.rs | 25 +- crates/spark/src/function/hash/crc32.rs | 21 +- crates/spark/src/function/hash/sha1.rs | 21 +- crates/spark/src/function/mod.rs | 1 + crates/spark/src/function/nullability.rs | 227 ++++++++++++++++++ crates/spark/src/function/string/ascii.rs | 26 +- crates/spark/src/function/string/char.rs | 20 +- crates/spark/src/function/string/length.rs | 21 +- crates/spark/src/function/string/substring.rs | 27 ++- .../sqllogictest/test_files/array/slice.slt | 8 + .../test_files/bitwise/bit_get.slt | 8 + .../test_files/datetime/date_add.slt | 8 + .../test_files/datetime/date_sub.slt | 8 + .../test_files/string/substring.slt | 21 ++ 17 files changed, 460 insertions(+), 48 deletions(-) create mode 100644 crates/spark/src/function/nullability.rs diff --git a/crates/spark/src/function/array/slice.rs b/crates/spark/src/function/array/slice.rs index 5e6f7274..9cba749d 100644 --- a/crates/spark/src/function/array/slice.rs +++ b/crates/spark/src/function/array/slice.rs @@ -9,6 +9,8 @@ use datafusion_common::utils::ListCoercion; use datafusion_common::{ Result, ScalarValue, exec_err, internal_err, utils::take_function_args, }; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, @@ -18,6 +20,11 @@ use datafusion_functions_nested::extract::array_slice_udf; use datafusion_macros::user_doc; use std::sync::Arc; +use crate::function::nullability::SparkNullability; + +/// Spark's `Slice` is a null-intolerant `TernaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `slice` expression. /// /// @@ -94,8 +101,6 @@ impl ScalarUDFImpl for SparkSlice { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - let data_type = match args.arg_fields[0].data_type() { DataType::Null => { DataType::List(Arc::new(Field::new_list_field(DataType::Null, true))) @@ -103,7 +108,15 @@ impl ScalarUDFImpl for SparkSlice { dt => dt.clone(), }; - Ok(Arc::new(Field::new("slice", data_type, nullable))) + Ok(NULLABILITY.return_field("slice", data_type, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/bitwise/bit_get.rs b/crates/spark/src/function/bitwise/bit_get.rs index 64b6e50a..6ff69811 100644 --- a/crates/spark/src/function/bitwise/bit_get.rs +++ b/crates/spark/src/function/bitwise/bit_get.rs @@ -10,10 +10,12 @@ use arrow::array::{ downcast_integer_array, }; use arrow::compute::try_binary; -use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef, Int8Type, Int32Type}; +use arrow::datatypes::{ArrowNativeType, DataType, FieldRef, Int8Type, Int32Type}; use datafusion_common::types::{NativeType, logical_int32}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -21,6 +23,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::SparkNullability; + +/// Spark's `BitwiseGet` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `bit_get` expression. /// /// @@ -97,9 +104,15 @@ impl ScalarUDFImpl for SparkBitGet { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - // Spark derives nullability for BinaryExpression from its children - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Int8, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int8, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -142,6 +155,7 @@ fn spark_bit_get(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_bit_get_nullability_non_nullable_inputs() { diff --git a/crates/spark/src/function/datetime/date_add.rs b/crates/spark/src/function/datetime/date_add.rs index acec7759..30799b4b 100644 --- a/crates/spark/src/function/datetime/date_add.rs +++ b/crates/spark/src/function/datetime/date_add.rs @@ -6,12 +6,14 @@ use std::sync::Arc; use arrow::array::ArrayRef; use arrow::compute; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef}; +use arrow::datatypes::{DataType, Date32Type, FieldRef}; use datafusion_common::cast::{ as_date32_array, as_int8_array, as_int16_array, as_int32_array, }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -19,6 +21,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::SparkNullability; + +/// Spark's `DateAdd` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `date_add` expression. /// /// @@ -95,12 +102,15 @@ impl ScalarUDFImpl for SparkDateAdd { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -153,6 +163,7 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_date_add_non_nullable_inputs() { diff --git a/crates/spark/src/function/datetime/date_sub.rs b/crates/spark/src/function/datetime/date_sub.rs index 3d5829f2..8c025bbd 100644 --- a/crates/spark/src/function/datetime/date_sub.rs +++ b/crates/spark/src/function/datetime/date_sub.rs @@ -6,11 +6,13 @@ use std::sync::Arc; use arrow::array::ArrayRef; use arrow::compute; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef}; +use arrow::datatypes::{DataType, Date32Type, FieldRef}; use datafusion_common::cast::{ as_date32_array, as_int8_array, as_int16_array, as_int32_array, }; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -18,6 +20,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::SparkNullability; + +/// Spark's `DateSub` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `date_sub` expression. /// /// @@ -87,12 +94,15 @@ impl ScalarUDFImpl for SparkDateSub { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -150,6 +160,7 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_date_sub_nullability_non_nullable_args() { diff --git a/crates/spark/src/function/hash/crc32.rs b/crates/spark/src/function/hash/crc32.rs index 9bc224e9..bf3adf75 100644 --- a/crates/spark/src/function/hash/crc32.rs +++ b/crates/spark/src/function/hash/crc32.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use arrow::array::{ArrayRef, Int64Array}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use crc32fast::Hasher; use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_fixed_size_binary_array, @@ -22,6 +22,13 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; + +use crate::function::nullability::SparkNullability; + +/// Spark's `Crc32` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `crc32` expression. /// @@ -91,8 +98,15 @@ impl ScalarUDFImpl for SparkCrc32 { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int64, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -147,6 +161,7 @@ fn spark_crc32(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_crc32_nullability() -> Result<()> { diff --git a/crates/spark/src/function/hash/sha1.rs b/crates/spark/src/function/hash/sha1.rs index 650baf02..20835c60 100644 --- a/crates/spark/src/function/hash/sha1.rs +++ b/crates/spark/src/function/hash/sha1.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use arrow::array::{ArrayRef, StringArray, StringBuilder}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_fixed_size_binary_array, as_large_binary_array, @@ -23,6 +23,13 @@ use datafusion_macros::user_doc; use sha1::{Digest, Sha1}; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; + +use crate::function::nullability::SparkNullability; + +/// Spark's `Sha1` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `sha1` expression. /// @@ -99,8 +106,15 @@ impl ScalarUDFImpl for SparkSha1 { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -163,6 +177,7 @@ fn spark_sha1(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_sha1_nullability() -> Result<()> { diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 1c165bed..a2fc12b2 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -23,6 +23,7 @@ pub mod map; pub mod math; pub mod misc; mod null_utils; +pub(crate) mod nullability; pub mod predicate; pub mod string; pub mod r#struct; diff --git a/crates/spark/src/function/nullability.rs b/crates/spark/src/function/nullability.rs new file mode 100644 index 00000000..ea1d094b --- /dev/null +++ b/crates/spark/src/function/nullability.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! A function's nullability, declared as Spark declares it. +//! +//! Spark describes it with two members of `Expression`: +//! +//! - `nullable`: whether the result can be NULL. `UnaryExpression`, `BinaryExpression` and +//! `TernaryExpression` take it from their children (`children.exists(_.nullable)`), and an +//! expression that can return NULL for non-NULL inputs overrides it, as `next_day` does. +//! - `nullIntolerant` (the `NullIntolerant` trait before Spark 4.0): a NULL input always gives a +//! NULL output. The `NullPropagation` optimizer rule uses it to replace a call that has a NULL +//! literal argument with a NULL literal. +//! +//! These are independent: a null-intolerant expression can still be nullable for non-NULL +//! inputs. +//! +//! A function here declares both in a [`SparkNullability`] and uses it from `ScalarUDFImpl`: +//! +//! ```ignore +//! const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); +//! +//! fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { +//! Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) +//! } +//! +//! fn simplify(&self, args: Vec, info: &SimplifyContext) -> Result { +//! NULLABILITY.simplify(self, args, info) +//! } +//! ``` + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::{Result, ScalarValue}; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use datafusion_expr::{Expr, ExprSchemable, ReturnFieldArgs, ScalarUDFImpl, lit}; + +/// When a function's result can be NULL: Spark's `Expression.nullable`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[allow( + dead_code, + reason = "declared for the functions that move onto SparkNullability next (#383)" +)] +pub(crate) enum SparkNullable { + /// When any argument can be NULL: `children.exists(_.nullable)`, the default of + /// `UnaryExpression`, `BinaryExpression` and `TernaryExpression`. + Children, + /// Always, because the function can return NULL for non-NULL arguments. + Always, + /// Never. + Never, +} + +/// A function's nullability: Spark's `nullable` and `nullIntolerant`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct SparkNullability { + nullable: SparkNullable, + null_intolerant: bool, +} + +impl SparkNullability { + /// A null-intolerant function whose result is nullable when an argument is. This is a + /// `UnaryExpression`, `BinaryExpression` or `TernaryExpression` with + /// `nullIntolerant = true` that does not override `nullable`. + pub(crate) const fn null_intolerant() -> Self { + Self::new(SparkNullable::Children, true) + } + + /// Declares `nullable` and `nullIntolerant` separately. + pub(crate) const fn new(nullable: SparkNullable, null_intolerant: bool) -> Self { + Self { + nullable, + null_intolerant, + } + } + + /// Whether the result can be NULL, given the argument fields. + pub(crate) fn nullable(&self, arg_fields: &[FieldRef]) -> bool { + match self.nullable { + SparkNullable::Children => arg_fields.iter().any(|f| f.is_nullable()), + SparkNullable::Always => true, + SparkNullable::Never => false, + } + } + + /// The return field, for `ScalarUDFImpl::return_field_from_args`. + pub(crate) fn return_field( + &self, + name: &str, + data_type: DataType, + arg_fields: &[FieldRef], + ) -> FieldRef { + Arc::new(Field::new(name, data_type, self.nullable(arg_fields))) + } + + /// For `ScalarUDFImpl::simplify`: replaces a call to a null-intolerant function that has + /// a NULL literal argument with a NULL literal of its return type, as Spark's + /// `NullPropagation` does. Otherwise returns the arguments unchanged. + /// + /// This is only an optimization: `invoke_with_args` must return NULL for such a call too. + pub(crate) fn simplify( + &self, + func: &F, + args: Vec, + info: &SimplifyContext, + ) -> Result { + if !self.null_intolerant || !args.iter().any(is_null_literal) { + return Ok(ExprSimplifyResult::Original(args)); + } + let arg_fields = args + .iter() + .map(|arg| Ok(arg.to_field(info.schema())?.1)) + .collect::>>()?; + let scalar_arguments = args + .iter() + .map(|arg| match arg { + Expr::Literal(value, _) => Some(value), + _ => None, + }) + .collect::>(); + let return_field = func.return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &scalar_arguments, + })?; + let null = ScalarValue::try_from(return_field.data_type())?; + Ok(ExprSimplifyResult::Simplified(lit(null))) + } +} + +fn is_null_literal(expr: &Expr) -> bool { + matches!(expr, Expr::Literal(value, _) if value.is_null()) +} + +#[cfg(test)] +mod tests { + use super::*; + + use datafusion_common::{DFSchema, ScalarValue}; + use datafusion_expr::col; + + use crate::function::string::ascii::SparkAscii; + + fn fields(nullable: &[bool]) -> Vec { + nullable + .iter() + .enumerate() + .map(|(i, n)| Arc::new(Field::new(format!("a{i}"), DataType::Utf8, *n))) + .collect() + } + + #[test] + fn test_nullable_from_children() { + let nullability = SparkNullability::null_intolerant(); + assert!(!nullability.nullable(&[])); + assert!(!nullability.nullable(&fields(&[false, false]))); + assert!(nullability.nullable(&fields(&[false, true]))); + } + + #[test] + fn test_nullable_always_and_never() { + let always = SparkNullability::new(SparkNullable::Always, true); + assert!(always.nullable(&fields(&[false]))); + let never = SparkNullability::new(SparkNullable::Never, false); + assert!(!never.nullable(&fields(&[true]))); + } + + #[test] + fn test_return_field() { + let field = SparkNullability::null_intolerant().return_field( + "f", + DataType::Int32, + &fields(&[true]), + ); + assert_eq!(field.as_ref(), &Field::new("f", DataType::Int32, true)); + } + + fn schema() -> SimplifyContext { + let schema = DFSchema::try_from(arrow::datatypes::Schema::new(vec![Field::new( + "s", + DataType::Utf8, + true, + )])) + .unwrap(); + SimplifyContext::builder() + .with_schema(Arc::new(schema)) + .build() + } + + #[test] + fn test_simplify_null_literal_to_typed_null() { + let result = SparkNullability::null_intolerant() + .simplify( + &SparkAscii::new(), + vec![lit(ScalarValue::Utf8(None))], + &schema(), + ) + .unwrap(); + let ExprSimplifyResult::Simplified(expr) = result else { + panic!("expected a NULL literal"); + }; + assert_eq!(expr, lit(ScalarValue::Int32(None))); + } + + #[test] + fn test_simplify_keeps_non_null_arguments() { + for args in [vec![col("s")], vec![lit("a")]] { + let result = SparkNullability::null_intolerant() + .simplify(&SparkAscii::new(), args.clone(), &schema()) + .unwrap(); + let ExprSimplifyResult::Original(original) = result else { + panic!("expected the arguments unchanged"); + }; + assert_eq!(original, args); + } + } + + #[test] + fn test_simplify_not_null_intolerant() { + let args = vec![lit(ScalarValue::Utf8(None))]; + let result = SparkNullability::new(SparkNullable::Children, false) + .simplify(&SparkAscii::new(), args.clone(), &schema()) + .unwrap(); + assert!(matches!(result, ExprSimplifyResult::Original(a) if a == args)); + } +} diff --git a/crates/spark/src/function/string/ascii.rs b/crates/spark/src/function/string/ascii.rs index 35e1af7a..33a0a78c 100644 --- a/crates/spark/src/function/string/ascii.rs +++ b/crates/spark/src/function/string/ascii.rs @@ -2,11 +2,11 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::types::{NativeType, logical_string}; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -15,6 +15,11 @@ use datafusion_functions::string::ascii::ascii; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::SparkNullability; + +/// Spark's `Ascii` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `ascii` expression. /// /// @@ -81,10 +86,15 @@ impl ScalarUDFImpl for SparkAscii { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - // ascii returns an Int32 value - // The result is nullable only if any of the input arguments is nullable - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new("ascii", DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -99,6 +109,8 @@ impl ScalarUDFImpl for SparkAscii { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; + use std::sync::Arc; #[test] fn test_return_field_nullable_input() { diff --git a/crates/spark/src/function/string/char.rs b/crates/spark/src/function/string/char.rs index bf378607..ea59c1f2 100644 --- a/crates/spark/src/function/string/char.rs +++ b/crates/spark/src/function/string/char.rs @@ -6,16 +6,23 @@ use arrow::array::ArrayRef; use arrow::array::GenericStringBuilder; use arrow::datatypes::DataType::Int64; use arrow::datatypes::DataType::Utf8; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_macros::user_doc; use std::sync::Arc; use datafusion_common::{Result, ScalarValue, cast::as_int64_array, exec_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; +use crate::function::nullability::SparkNullability; + +/// Spark's `Chr` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `char` expression. /// /// @@ -81,8 +88,15 @@ impl ScalarUDFImpl for CharFunc { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), Utf8, nullable))) + Ok(NULLABILITY.return_field(self.name(), Utf8, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } } diff --git a/crates/spark/src/function/string/length.rs b/crates/spark/src/function/string/length.rs index 632fbe91..cd1b49f4 100644 --- a/crates/spark/src/function/string/length.rs +++ b/crates/spark/src/function/string/length.rs @@ -5,8 +5,10 @@ use arrow::array::{ Array, ArrayRef, AsArray, BinaryArrayType, PrimitiveArray, StringArrayType, }; -use arrow::datatypes::{DataType, Field, FieldRef, Int32Type}; +use arrow::datatypes::{DataType, FieldRef, Int32Type}; use datafusion_common::exec_err; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -15,6 +17,11 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +use crate::function::nullability::SparkNullability; + +/// Spark's `Length` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `length` expression. /// /// @@ -109,9 +116,16 @@ impl ScalarUDFImpl for SparkLengthFunc { &self, args: ReturnFieldArgs, ) -> datafusion_common::Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); // spark length always returns Int32 - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> datafusion_common::Result { + NULLABILITY.simplify(self, args, info) } } @@ -214,6 +228,7 @@ mod tests { use crate::function::utils::test::test_scalar_function; use arrow::array::Int32Array; use arrow::datatypes::DataType::Int32; + use arrow::datatypes::Field; use datafusion_common::{Result, ScalarValue}; macro_rules! test_spark_length_string { diff --git a/crates/spark/src/function/string/substring.rs b/crates/spark/src/function/string/substring.rs index 3fda3918..4e6dc519 100644 --- a/crates/spark/src/function/string/substring.rs +++ b/crates/spark/src/function/string/substring.rs @@ -8,12 +8,14 @@ use arrow::array::{ StringViewBuilder, }; use arrow::datatypes::DataType; -use datafusion_common::arrow::datatypes::{Field, FieldRef}; +use datafusion_common::arrow::datatypes::FieldRef; use datafusion_common::cast::as_int64_array; use datafusion_common::types::{ NativeType, logical_int32, logical_int64, logical_string, }; use datafusion_common::{Result, exec_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{Coercion, Documentation, ReturnFieldArgs, TypeSignatureClass}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, @@ -22,7 +24,11 @@ use datafusion_expr::{ use datafusion_functions::unicode::substr::{enable_ascii_fast_path, get_true_start_end}; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; -use std::sync::Arc; + +use crate::function::nullability::SparkNullability; + +/// Spark's `Substring` is a null-intolerant `TernaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `substring` expression /// @@ -142,14 +148,19 @@ impl ScalarUDFImpl for SparkSubstring { } fn return_field_from_args(&self, args: ReturnFieldArgs<'_>) -> Result { - // Spark semantics: substring returns NULL if ANY input is NULL - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - - Ok(Arc::new(Field::new( + Ok(NULLABILITY.return_field( "substring", args.arg_fields[0].data_type().clone(), - nullable, - ))) + args.arg_fields, + )) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } } diff --git a/crates/sqllogictest/test_files/array/slice.slt b/crates/sqllogictest/test_files/array/slice.slt index e7f50a0d..368b2389 100644 --- a/crates/sqllogictest/test_files/array/slice.slt +++ b/crates/sqllogictest/test_files/array/slice.slt @@ -145,3 +145,11 @@ query ?T SELECT slice(array(1, 2, 3, 4), 2, 2), arrow_typeof(slice(array(1, 2, 3, 4), 2, 2)); ---- [2, 3] List(Int64, field: 'element') + +# slice is null-intolerant, as in Spark: a NULL argument makes every row NULL +query ?? +SELECT slice(column1, NULL::int, 2::int), slice(column1, 1::int, NULL::int) +FROM VALUES (array(1, 2, 3)), (NULL); +---- +NULL NULL +NULL NULL diff --git a/crates/sqllogictest/test_files/bitwise/bit_get.slt b/crates/sqllogictest/test_files/bitwise/bit_get.slt index d93a827d..76f508a6 100644 --- a/crates/sqllogictest/test_files/bitwise/bit_get.slt +++ b/crates/sqllogictest/test_files/bitwise/bit_get.slt @@ -85,3 +85,11 @@ query I SELECT bit_get(11, 3::bigint); ---- 1 + +# bit_get is null-intolerant, as in Spark: a NULL argument makes every row NULL +query I +SELECT bit_get(column1, NULL::int) +FROM VALUES (11), (NULL); +---- +NULL +NULL diff --git a/crates/sqllogictest/test_files/datetime/date_add.slt b/crates/sqllogictest/test_files/datetime/date_add.slt index 385a4a23..82298e81 100644 --- a/crates/sqllogictest/test_files/datetime/date_add.slt +++ b/crates/sqllogictest/test_files/datetime/date_add.slt @@ -80,3 +80,11 @@ SELECT date_add('2024-01-01', 1::INT); # xfail spark: An integer literal is BIGINT here and INT in Spark (https://github.com/apache/datafusion-spark/issues/97) statement error Failed to coerce arguments to satisfy a call to 'date_add' function SELECT date_add(DATE '2024-01-01', 1); + +# date_add is null-intolerant, as in Spark: a NULL argument makes every row NULL +query D +SELECT date_add(column1, NULL::int) +FROM VALUES (DATE '2024-01-01'), (NULL); +---- +NULL +NULL diff --git a/crates/sqllogictest/test_files/datetime/date_sub.slt b/crates/sqllogictest/test_files/datetime/date_sub.slt index 11806234..8bcf7ae0 100644 --- a/crates/sqllogictest/test_files/datetime/date_sub.slt +++ b/crates/sqllogictest/test_files/datetime/date_sub.slt @@ -79,3 +79,11 @@ SELECT date_sub('2024-01-01', 1::INT); # xfail spark: An integer literal is BIGINT here and INT in Spark (https://github.com/apache/datafusion-spark/issues/97) statement error Failed to coerce arguments to satisfy a call to 'date_sub' function SELECT date_sub(DATE '2024-01-01', 1); + +# date_sub is null-intolerant, as in Spark: a NULL argument makes every row NULL +query D +SELECT date_sub(column1, NULL::int) +FROM VALUES (DATE '2024-01-01'), (NULL); +---- +NULL +NULL diff --git a/crates/sqllogictest/test_files/string/substring.slt b/crates/sqllogictest/test_files/string/substring.slt index 153575f6..58af8549 100644 --- a/crates/sqllogictest/test_files/string/substring.slt +++ b/crates/sqllogictest/test_files/string/substring.slt @@ -360,3 +360,24 @@ FROM VALUES NULL NULL NULL NULL NULL NULL + +# substring is null-intolerant, as in Spark: a NULL argument makes every row NULL +query TT +SELECT substring(column1, NULL::int), substring(column1, 2::int, NULL::int) +FROM VALUES ('Spark SQL'::string), (NULL::string); +---- +NULL NULL +NULL NULL + +# The simplifier replaces a call with a NULL literal argument with a NULL literal, as +# Spark's NullPropagation does. Without the simplifier the call is planned as written. +skipif evaluate +query TT +EXPLAIN SELECT substring(column1, NULL::int) FROM VALUES ('Spark SQL'::string); +---- +logical_plan +01)Projection: Utf8View(NULL) AS substring(column1,NULL) +02)--Values: (Utf8View("Spark SQL") AS Utf8("Spark SQL")) +physical_plan +01)ProjectionExec: expr=[NULL as substring(column1,NULL)] +02)--DataSourceExec: partitions=1, partition_sizes=[1] From 3aa0abccbe8736492f8802a1c8df58169d0025f5 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 07:23:40 -0700 Subject: [PATCH 2/5] feat(spark): move the remaining null-intolerant functions onto SparkNullability Declare Spark's nullable and nullIntolerant for every function that is null-intolerant in Spark, so that a call with a NULL literal argument folds to a NULL literal and the result field follows Spark's nullability. Functions that only implemented return_type were always nullable. Those whose Spark expression takes nullability from its children are now non-nullable for non-nullable arguments: array_position, array_remove, atan2, contains, hex, hour, minute, second, hypot, negative, pow, soundex and space. floor, make_valid_utf8 and the functions Spark marks as always nullable are now always nullable. make_dt_interval is always nullable, because an overflow returns NULL (#20). This fixes #315, so its records no longer skip evaluate mode. Part of #383. --- .../src/function/array/array_contains.rs | 33 ++++++++++++- .../spark/src/function/array/array_extrema.rs | 32 +++++++++++++ .../src/function/array/array_position.rs | 20 ++++++++ .../spark/src/function/array/array_remove.rs | 20 ++++++++ .../spark/src/function/bitwise/bit_count.rs | 22 ++++++--- .../spark/src/function/bitwise/bit_shift.rs | 22 +++++++-- .../spark/src/function/bitwise/bitwise_not.rs | 26 ++++++---- .../spark/src/function/datetime/add_months.rs | 19 ++++---- .../spark/src/function/datetime/date_diff.rs | 15 ++++-- .../function/datetime/date_from_unix_date.rs | 23 ++++++--- crates/spark/src/function/datetime/dayname.rs | 21 ++++++-- crates/spark/src/function/datetime/extract.rs | 44 +++++++++++++++++ .../function/datetime/from_utc_timestamp.rs | 21 +++++--- .../spark/src/function/datetime/last_day.rs | 25 ++++++---- .../spark/src/function/datetime/make_date.rs | 26 ++++++++-- .../src/function/datetime/make_dt_interval.rs | 48 ++++++++++--------- .../src/function/datetime/make_interval.rs | 23 +++++++++ .../spark/src/function/datetime/monthname.rs | 21 ++++++-- .../spark/src/function/datetime/next_day.rs | 24 ++++++++-- .../src/function/datetime/to_utc_timestamp.rs | 23 ++++++--- crates/spark/src/function/datetime/unix.rs | 11 ++++- crates/spark/src/function/datetime/weekday.rs | 23 +++++++-- crates/spark/src/function/hash/sha2.rs | 21 ++++++++ .../spark/src/function/map/map_from_arrays.rs | 22 +++++++-- .../src/function/map/map_from_entries.rs | 18 ++++++- crates/spark/src/function/map/str_to_map.rs | 19 ++++++-- crates/spark/src/function/math/abs.rs | 22 +++++++-- crates/spark/src/function/math/atan2.rs | 20 ++++++++ crates/spark/src/function/math/bin.rs | 22 ++++++--- crates/spark/src/function/math/ceil.rs | 20 ++++++++ crates/spark/src/function/math/divide.rs | 21 ++++++++ crates/spark/src/function/math/expm1.rs | 20 ++++++++ crates/spark/src/function/math/factorial.rs | 21 ++++++++ crates/spark/src/function/math/floor.rs | 19 ++++++-- crates/spark/src/function/math/hex.rs | 23 +++++++++ crates/spark/src/function/math/hypot.rs | 20 ++++++++ crates/spark/src/function/math/modulus.rs | 33 +++++++++++++ crates/spark/src/function/math/negative.rs | 20 ++++++++ crates/spark/src/function/math/pow.rs | 20 ++++++++ crates/spark/src/function/math/rint.rs | 20 ++++++++ crates/spark/src/function/math/unhex.rs | 21 ++++++++ .../spark/src/function/math/width_bucket.rs | 21 ++++++++ crates/spark/src/function/nullability.rs | 43 ++++++++++++----- crates/spark/src/function/string/base64.rs | 18 +++++-- crates/spark/src/function/string/contains.rs | 20 ++++++++ .../src/function/string/is_valid_utf8.rs | 20 ++++++-- crates/spark/src/function/string/like.rs | 24 +++++++--- .../src/function/string/make_valid_utf8.rs | 22 ++++++--- crates/spark/src/function/string/soundex.rs | 20 ++++++++ crates/spark/src/function/string/space.rs | 20 ++++++++ .../test_files/datetime/make_dt_interval.slt | 21 -------- crates/sqllogictest/test_files/math/atan2.slt | 8 ++++ crates/sqllogictest/test_files/math/pmod.slt | 8 ++++ .../test_files/predicate/like.slt | 8 ++++ .../test_files/string/contains.slt | 8 ++++ 55 files changed, 1016 insertions(+), 189 deletions(-) diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 029e6c09..7f1e9886 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -3,18 +3,26 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_floats; +use crate::function::nullability::SparkNullability; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; use arrow::buffer::{BooleanBuffer, NullBuffer}; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{Result, exec_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_functions_nested::array_has::array_has_udf; use std::sync::Arc; +/// Spark's `ArrayContains` is a null-intolerant `BinaryExpression`. It is also nullable +/// when the array can hold NULL elements. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `array_contains` function. /// /// Calls DataFusion's `array_has` and then applies Spark's null semantics: @@ -59,6 +67,29 @@ impl ScalarUDFImpl for SparkArrayContains { Ok(DataType::Boolean) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let contains_null = match args.arg_fields[0].data_type() { + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => field.is_nullable(), + _ => true, + }; + let nullable = NULLABILITY.nullable(args.arg_fields) || contains_null; + Ok(Arc::new(Field::new( + self.name(), + DataType::Boolean, + nullable, + ))) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, mut args: ScalarFunctionArgs) -> Result { for arg in &mut args.args { *arg = normalize_floats(arg)?; diff --git a/crates/spark/src/function/array/array_extrema.rs b/crates/spark/src/function/array/array_extrema.rs index 28cdbad4..fa9d03d4 100644 --- a/crates/spark/src/function/array/array_extrema.rs +++ b/crates/spark/src/function/array/array_extrema.rs @@ -5,6 +5,7 @@ //! Spark-compatible `array_max` and `array_min`. use crate::function::float_normalize::normalize_array; +use crate::function::nullability::SparkNullability; use crate::function::type_coercion::{is_orderable, spark_sql_type}; use arrow::array::{ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray, OffsetSizeTrait, @@ -18,6 +19,9 @@ use arrow::datatypes::{ }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -26,6 +30,10 @@ use datafusion_macros::user_doc; use std::cmp::Ordering; use std::sync::Arc; +/// Spark's `ArrayMax` and `ArrayMin` are null-intolerant and always nullable, because an +/// empty array returns NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `array_max` expression. /// /// @@ -82,6 +90,18 @@ impl ScalarUDFImpl for SparkArrayMax { element_type(self.name(), arg_types) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { coerce_array(self.name(), arg_types) } @@ -151,6 +171,18 @@ impl ScalarUDFImpl for SparkArrayMin { element_type(self.name(), arg_types) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { coerce_array(self.name(), arg_types) } diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 445a8b48..113942ab 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_array; +use crate::function::nullability::SparkNullability; use crate::function::type_coercion::{ is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; @@ -11,9 +12,13 @@ use arrow::array::{ }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -21,6 +26,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `ArrayPosition` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `array_position` expression. /// /// @@ -85,6 +93,18 @@ impl ScalarUDFImpl for SparkArrayPosition { Ok(DataType::Int64) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { let [array, element] = take_function_args(self.name(), arg_types)?; coerce_array_and_element(self.name(), array, element) diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs index 77b1da94..04e976c0 100644 --- a/crates/spark/src/function/array/array_remove.rs +++ b/crates/spark/src/function/array/array_remove.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_array; +use crate::function::nullability::SparkNullability; use crate::function::type_coercion::{ is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; @@ -13,9 +14,13 @@ use arrow::array::{ use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer}; use arrow::compute::kernels::cmp::eq; use arrow::compute::{SortOptions, filter}; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Field, Float32Type, Float64Type}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -24,6 +29,9 @@ use datafusion_macros::user_doc; use num_traits::Float; use std::sync::Arc; +/// Spark's `ArrayRemove` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `array_remove` expression. /// /// @@ -85,6 +93,18 @@ impl ScalarUDFImpl for SparkArrayRemove { Ok(array.clone()) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { let [array, element] = take_function_args(self.name(), arg_types)?; coerce_array_and_element(self.name(), array, element) diff --git a/crates/spark/src/function/bitwise/bit_count.rs b/crates/spark/src/function/bitwise/bit_count.rs index 9c63274a..1f421971 100644 --- a/crates/spark/src/function/bitwise/bit_count.rs +++ b/crates/spark/src/function/bitwise/bit_count.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, AsArray, Int32Array}; use arrow::datatypes::{ DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, @@ -11,6 +12,8 @@ use arrow::datatypes::{ }; use datafusion_common::cast::as_boolean_array; use datafusion_common::{Result, internal_err, plan_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -18,6 +21,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `BitwiseCount` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `bit_count` expression. /// /// @@ -91,13 +97,15 @@ impl ScalarUDFImpl for SparkBitCount { &self, args: datafusion_expr::ReturnFieldArgs, ) -> Result { - use arrow::datatypes::Field; - // bit_count returns Int32 with the same nullability as the input - Ok(Arc::new(Field::new( - args.arg_fields[0].name(), - DataType::Int32, - args.arg_fields[0].is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/bitwise/bit_shift.rs b/crates/spark/src/function/bitwise/bit_shift.rs index a50d7282..322efdd0 100644 --- a/crates/spark/src/function/bitwise/bit_shift.rs +++ b/crates/spark/src/function/bitwise/bit_shift.rs @@ -4,11 +4,11 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, Int32Array, PrimitiveArray}; use arrow::compute; use arrow::datatypes::{ - ArrowNativeType, DataType, Field, FieldRef, Int32Type, Int64Type, UInt32Type, - UInt64Type, + ArrowNativeType, DataType, FieldRef, Int32Type, Int64Type, UInt32Type, UInt64Type, }; use datafusion_common::types::{ NativeType, logical_int8, logical_int16, logical_int32, logical_int64, logical_uint8, @@ -16,6 +16,8 @@ use datafusion_common::types::{ }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -180,6 +182,10 @@ enum BitShiftType { RightUnsigned, } +/// Spark's `ShiftLeft`, `ShiftRight` and `ShiftRightUnsigned` are null-intolerant +/// `BinaryExpression`s. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `shiftleft`, `shiftright` and `shiftrightunsigned` expressions. /// /// One struct implements all three functions, so each function's documentation is @@ -353,9 +359,16 @@ impl ScalarUDFImpl for SparkBitShift { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); let data_type = args.arg_fields[0].data_type().clone(); - Ok(Arc::new(Field::new(self.name(), data_type, nullable))) + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -377,6 +390,7 @@ impl ScalarUDFImpl for SparkBitShift { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; #[test] fn test_bit_shift_nullability() -> Result<()> { diff --git a/crates/spark/src/function/bitwise/bitwise_not.rs b/crates/spark/src/function/bitwise/bitwise_not.rs index 758fc321..7e34e262 100644 --- a/crates/spark/src/function/bitwise/bitwise_not.rs +++ b/crates/spark/src/function/bitwise/bitwise_not.rs @@ -2,18 +2,22 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::*; use arrow::compute::kernels::bitwise; -use arrow::datatypes::{ - DataType, Field, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type, -}; +use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type}; use datafusion_common::{Result, internal_err, plan_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ColumnarValue, Documentation, TypeSignature, Volatility}; use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature}; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `BitwiseNot` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `bitwise_not` expression (the `~` operator in Spark SQL). /// /// @@ -79,11 +83,16 @@ impl ScalarUDFImpl for SparkBitwiseNot { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - Ok(Arc::new(Field::new( - self.name(), - args.arg_fields[0].data_type().clone(), - args.arg_fields[0].is_nullable(), - ))) + let data_type = args.arg_fields[0].data_type().clone(); + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -133,6 +142,7 @@ pub fn spark_bitwise_not(args: &[ArrayRef]) -> Result { #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; use std::sync::Arc; #[test] diff --git a/crates/spark/src/function/datetime/add_months.rs b/crates/spark/src/function/datetime/add_months.rs index c8d28789..b1888f52 100644 --- a/crates/spark/src/function/datetime/add_months.rs +++ b/crates/spark/src/function/datetime/add_months.rs @@ -3,9 +3,9 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use std::ops::Add; -use std::sync::Arc; -use arrow::datatypes::{DataType, Field, FieldRef, IntervalUnit}; +use crate::function::nullability::SparkNullability; +use arrow::datatypes::{DataType, FieldRef, IntervalUnit}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; @@ -15,6 +15,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `AddMonths` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `add_months` expression. /// /// @@ -81,13 +84,7 @@ impl ScalarUDFImpl for SparkAddMonths { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } fn documentation(&self) -> Option<&Documentation> { @@ -99,6 +96,10 @@ impl ScalarUDFImpl for SparkAddMonths { args: Vec, info: &SimplifyContext, ) -> Result { + let args = match NULLABILITY.simplify(self, args, info)? { + ExprSimplifyResult::Original(args) => args, + simplified => return Ok(simplified), + }; let [date_arg, months_arg] = take_function_args("add_months", args)?; let interval = months_arg .cast_to(&DataType::Interval(IntervalUnit::YearMonth), info.schema())?; diff --git a/crates/spark/src/function/datetime/date_diff.rs b/crates/spark/src/function/datetime/date_diff.rs index f593b566..acd1299d 100644 --- a/crates/spark/src/function/datetime/date_diff.rs +++ b/crates/spark/src/function/datetime/date_diff.rs @@ -2,9 +2,8 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - -use arrow::datatypes::{DataType, Field, FieldRef}; +use crate::function::nullability::SparkNullability; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::types::{NativeType, logical_date, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; @@ -16,6 +15,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `DateDiff` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `date_diff` expression. /// /// @@ -104,8 +106,7 @@ impl ScalarUDFImpl for SparkDateDiff { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } fn documentation(&self) -> Option<&Documentation> { @@ -123,6 +124,10 @@ impl ScalarUDFImpl for SparkDateDiff { args: Vec, info: &SimplifyContext, ) -> Result { + let args = match NULLABILITY.simplify(self, args, info)? { + ExprSimplifyResult::Original(args) => args, + simplified => return Ok(simplified), + }; let [end, start] = take_function_args(self.name(), args)?; let end = end.cast_to(&DataType::Date32, info.schema())?; let start = start.cast_to(&DataType::Date32, info.schema())?; diff --git a/crates/spark/src/function/datetime/date_from_unix_date.rs b/crates/spark/src/function/datetime/date_from_unix_date.rs index deed4a21..a7066a44 100644 --- a/crates/spark/src/function/datetime/date_from_unix_date.rs +++ b/crates/spark/src/function/datetime/date_from_unix_date.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use arrow::array::{Array, Date32Array}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::as_int32_array; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; @@ -16,6 +16,12 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::int32_coercion; +use crate::function::nullability::SparkNullability; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; + +/// Spark's `DateFromUnixDate` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `date_from_unix_date` expression. /// @@ -81,12 +87,15 @@ impl ScalarUDFImpl for SparkDateFromUnixDate { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields[0].is_nullable(); - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/datetime/dayname.rs b/crates/spark/src/function/datetime/dayname.rs index 48544a34..dc5c1b2b 100644 --- a/crates/spark/src/function/datetime/dayname.rs +++ b/crates/spark/src/function/datetime/dayname.rs @@ -2,10 +2,8 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - use arrow::compute::DatePart; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -15,6 +13,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::{date_part_name, date_signature}; +use crate::function::nullability::SparkNullability; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; const DAY_NAMES: [&str; 7] = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]; @@ -23,6 +24,9 @@ fn day_number_to_name(day: i32) -> Option<&'static str> { DAY_NAMES.get(day as usize).copied() } +/// Spark's `DayName` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `dayname` expression (Spark 4.0+). /// /// @@ -84,8 +88,15 @@ impl ScalarUDFImpl for SparkDayName { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/datetime/extract.rs b/crates/spark/src/function/datetime/extract.rs index 200d621b..658b810c 100644 --- a/crates/spark/src/function/datetime/extract.rs +++ b/crates/spark/src/function/datetime/extract.rs @@ -2,11 +2,16 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::ArrayRef; use arrow::compute::{DatePart, date_part}; use arrow::datatypes::DataType; +use arrow::datatypes::FieldRef; use datafusion_common::Result; use datafusion_common::utils::take_function_args; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -26,6 +31,9 @@ fn extract_signature() -> Signature { // SparkHour // ----------------------------------------------------------------------------- +/// Spark's `Hour`, `Minute` and `Second` are null-intolerant `UnaryExpression`s. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `hour` expression. /// /// @@ -78,6 +86,18 @@ impl ScalarUDFImpl for SparkHour { Ok(DataType::Int32) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -149,6 +169,18 @@ impl ScalarUDFImpl for SparkMinute { Ok(DataType::Int32) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -221,6 +253,18 @@ impl ScalarUDFImpl for SparkSecond { Ok(DataType::Int32) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/datetime/from_utc_timestamp.rs b/crates/spark/src/function/datetime/from_utc_timestamp.rs index f40dba59..7a32450b 100644 --- a/crates/spark/src/function/datetime/from_utc_timestamp.rs +++ b/crates/spark/src/function/datetime/from_utc_timestamp.rs @@ -7,6 +7,7 @@ use std::hash::{Hash, Hasher}; use std::sync::Arc; use super::timezone::TimeZoneRules; +use crate::function::nullability::SparkNullability; use arrow::array::{ Array, ArrayRef, AsArray, BooleanArray, PrimitiveBuilder, StringArrayType, new_empty_array, @@ -20,6 +21,8 @@ use arrow::record_batch::RecordBatch; use datafusion_common::types::{NativeType, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_datafusion_err, exec_err, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ExpressionPlacement, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -28,6 +31,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; +/// Spark's `FromUTCTimestamp` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `from_utc_timestamp` expression. /// /// This UDF operates on already-evaluated arguments. Callers that need conditional @@ -113,13 +119,16 @@ impl ScalarUDFImpl for SparkFromUtcTimestamp { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); + let data_type = args.arg_fields[0].data_type().clone(); + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) + } - Ok(Arc::new(Field::new( - self.name(), - args.arg_fields[0].data_type().clone(), - nullable, - ))) + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { diff --git a/crates/spark/src/function/datetime/last_day.rs b/crates/spark/src/function/datetime/last_day.rs index 84029680..f705b6d6 100644 --- a/crates/spark/src/function/datetime/last_day.rs +++ b/crates/spark/src/function/datetime/last_day.rs @@ -4,17 +4,23 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, AsArray, Date32Array}; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef}; +use arrow::datatypes::{DataType, Date32Type, FieldRef}; use chrono::{Datelike, Duration, NaiveDate}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_datafusion_err, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_macros::user_doc; +/// Spark's `LastDay` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `last_day` expression. /// /// @@ -67,15 +73,15 @@ impl ScalarUDFImpl for SparkLastDay { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let Some(field) = args.arg_fields.first() else { - return internal_err!("Spark `last_day` expects exactly one argument"); - }; + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) + } - Ok(Arc::new(Field::new( - self.name(), - DataType::Date32, - field.is_nullable(), - ))) + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -150,6 +156,7 @@ mod tests { use super::*; use crate::function::utils::test::test_scalar_function; use arrow::array::Array; + use arrow::datatypes::Field; #[test] fn test_last_day_nullability_matches_input() { diff --git a/crates/spark/src/function/datetime/make_date.rs b/crates/spark/src/function/datetime/make_date.rs index d8e020cb..781c2089 100644 --- a/crates/spark/src/function/datetime/make_date.rs +++ b/crates/spark/src/function/datetime/make_date.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use arrow::array::{Array, Date32Array, Date32Builder, new_empty_array}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::as_int32_array; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; @@ -16,9 +16,18 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::int32_coercion; +use crate::function::nullability::SparkNullability; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. SQLSTATE: 22003"#; +/// Spark's `MakeDate` is null-intolerant. It is always nullable when ANSI mode is off, +/// because an invalid date returns NULL, and nullable when an argument is when ANSI mode is +/// on. ANSI mode is not known in `return_field_from_args`, so the result is always nullable +/// here. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `make_date` expression. /// /// @@ -97,10 +106,16 @@ impl ScalarUDFImpl for SparkMakeDate { internal_err!("return_field_from_args should be used instead") } - fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { - // An invalid date returns NULL when ANSI mode is off, which is not known - // at planning time, so the result is always nullable. - Ok(Arc::new(Field::new(self.name(), DataType::Date32, true))) + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -279,6 +294,7 @@ fn days_from_civil(year: i32, month: i32, day: i32) -> i64 { mod tests { use super::*; use arrow::array::Int32Array; + use arrow::datatypes::Field; use datafusion_common::config::ConfigOptions; fn invoke( diff --git a/crates/spark/src/function/datetime/make_dt_interval.rs b/crates/spark/src/function/datetime/make_dt_interval.rs index 87dc9d5f..c58d73ca 100644 --- a/crates/spark/src/function/datetime/make_dt_interval.rs +++ b/crates/spark/src/function/datetime/make_dt_interval.rs @@ -4,15 +4,18 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ Array, ArrayRef, AsArray, DurationMicrosecondBuilder, PrimitiveArray, }; use arrow::datatypes::TimeUnit::Microsecond; -use arrow::datatypes::{DataType, Field, FieldRef, Float64Type, Int32Type}; +use arrow::datatypes::{DataType, FieldRef, Float64Type, Int32Type}; use datafusion_common::types::{NativeType, logical_float64, logical_int32}; use datafusion_common::{ DataFusionError, Result, ScalarValue, internal_err, plan_datafusion_err, }; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -20,6 +23,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `MakeDTInterval` is a null-intolerant `QuaternaryExpression` that is nullable +/// when an argument is, but here an overflow returns NULL (#20), so the result is always +/// nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `make_dt_interval` expression. /// /// @@ -130,24 +138,19 @@ impl ScalarUDFImpl for SparkMakeDtInterval { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let has_non_finite_secs = args - .scalar_arguments - .get(3) - .and_then(|arg| { - arg.map(|scalar| match scalar { - ScalarValue::Float64(Some(v)) => !v.is_finite(), - ScalarValue::Float32(Some(v)) => !v.is_finite(), - _ => false, - }) - }) - .unwrap_or(false); - let nullable = - has_non_finite_secs || args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new( + Ok(NULLABILITY.return_field( self.name(), DataType::Duration(Microsecond), - nullable, - ))) + args.arg_fields, + )) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { @@ -284,6 +287,7 @@ mod tests { use datafusion_common::internal_datafusion_err; use super::*; + use arrow::datatypes::Field; fn run_make_dt_interval(arrs: Vec) -> Result { make_dt_interval_kernel(&arrs) @@ -364,7 +368,7 @@ mod tests { assert!(out.is_nullable()); assert_eq!(out.data_type(), &Duration(Microsecond)); - // Non-nullable inputs -> non-nullable output + // Non-nullable inputs -> nullable output, because an overflow returns NULL (#20) let non_nullable_arg_fields = vec![ Arc::new(Field::new("days", DataType::Int32, false)), Arc::new(Field::new("hours", DataType::Int32, false)), @@ -376,9 +380,9 @@ mod tests { arg_fields: &non_nullable_arg_fields, scalar_arguments: &[None, None, None, None], })?; - assert!(!out.is_nullable()); + assert!(out.is_nullable()); - // Non-finite secs scalar should force nullable even if fields are non-nullable + // Non-finite secs scalar is nullable too let scalar_values = [None, None, None, Some(ScalarValue::Float64(Some(f64::NAN)))]; let scalar_refs = scalar_values.iter().map(|v| v.as_ref()).collect::>(); @@ -388,12 +392,12 @@ mod tests { })?; assert!(out.is_nullable()); - // Zero-arg call (defaults) should also be non-nullable + // A zero-arg call (defaults) is nullable too, as the rule ignores the arguments let out = udf.return_field_from_args(ReturnFieldArgs { arg_fields: &[], scalar_arguments: &[], })?; - assert!(!out.is_nullable()); + assert!(out.is_nullable()); Ok(()) } diff --git a/crates/spark/src/function/datetime/make_interval.rs b/crates/spark/src/function/datetime/make_interval.rs index 88bfdaee..80f97322 100644 --- a/crates/spark/src/function/datetime/make_interval.rs +++ b/crates/spark/src/function/datetime/make_interval.rs @@ -4,12 +4,17 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{Array, ArrayRef, IntervalMonthDayNanoBuilder, PrimitiveArray}; use arrow::datatypes::DataType::Interval; +use arrow::datatypes::FieldRef; use arrow::datatypes::IntervalUnit::MonthDayNano; use arrow::datatypes::{DataType, IntervalMonthDayNano}; use datafusion_common::types::{NativeType, logical_float64, logical_int32}; use datafusion_common::{DataFusionError, Result, ScalarValue, plan_datafusion_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -17,6 +22,12 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `MakeInterval` is null-intolerant. It is always nullable when ANSI mode is off, +/// because an overflow returns NULL, and nullable when an argument is when ANSI mode is on. +/// ANSI mode is not known in `return_field_from_args`, so the result is always nullable +/// here. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `make_interval` expression. /// /// @@ -156,6 +167,18 @@ impl ScalarUDFImpl for SparkMakeInterval { Ok(Interval(MonthDayNano)) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.is_empty() { return Ok(ColumnarValue::Scalar(ScalarValue::IntervalMonthDayNano( diff --git a/crates/spark/src/function/datetime/monthname.rs b/crates/spark/src/function/datetime/monthname.rs index 7aa6a17c..820ec164 100644 --- a/crates/spark/src/function/datetime/monthname.rs +++ b/crates/spark/src/function/datetime/monthname.rs @@ -2,10 +2,8 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - use arrow::compute::DatePart; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -15,6 +13,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::{date_part_name, date_signature}; +use crate::function::nullability::SparkNullability; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; const MONTH_NAMES: [&str; 12] = [ "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", @@ -24,6 +25,9 @@ fn month_number_to_name(month: i32) -> Option<&'static str> { MONTH_NAMES.get((month - 1) as usize).copied() } +/// Spark's `MonthName` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `monthname` expression (Spark 4.0+). /// /// @@ -85,8 +89,15 @@ impl ScalarUDFImpl for SparkMonthName { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/datetime/next_day.rs b/crates/spark/src/function/datetime/next_day.rs index f386e86a..0de9dd12 100644 --- a/crates/spark/src/function/datetime/next_day.rs +++ b/crates/spark/src/function/datetime/next_day.rs @@ -4,19 +4,26 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ Array, ArrayRef, AsArray, Date32Array, Date32Builder, StringArrayType, }; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef}; +use arrow::datatypes::{DataType, Date32Type, FieldRef}; use chrono::{Datelike, Weekday}; use datafusion_common::types::{NativeType, logical_date, logical_string}; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; +/// Spark's `NextDay` is null-intolerant and always nullable, because an unknown day of the +/// week returns NULL when ANSI mode is off. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `next_day` expression. /// /// @@ -100,10 +107,16 @@ impl ScalarUDFImpl for SparkNextDay { internal_err!("return_field_from_args should be used instead") } - fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { - // Spark marks next_day as always nullable because invalid day_of_week values - // can yield NULL even when inputs are non-null. - Ok(Arc::new(Field::new(self.name(), DataType::Date32, true))) + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -373,6 +386,7 @@ fn spark_next_day( #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; use datafusion_common::config::ConfigOptions; #[test] diff --git a/crates/spark/src/function/datetime/to_utc_timestamp.rs b/crates/spark/src/function/datetime/to_utc_timestamp.rs index 674accbe..16f60181 100644 --- a/crates/spark/src/function/datetime/to_utc_timestamp.rs +++ b/crates/spark/src/function/datetime/to_utc_timestamp.rs @@ -4,11 +4,12 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::timezone::Tz; use arrow::array::{Array, ArrayRef, AsArray, PrimitiveBuilder, StringArrayType}; use arrow::datatypes::TimeUnit; use arrow::datatypes::{ - ArrowTimestampType, DataType, Field, FieldRef, TimestampMicrosecondType, + ArrowTimestampType, DataType, FieldRef, TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; use chrono::{DateTime, Offset, TimeZone}; @@ -17,6 +18,8 @@ use datafusion_common::utils::take_function_args; use datafusion_common::{ Result, exec_datafusion_err, exec_err, internal_datafusion_err, internal_err, }; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -24,6 +27,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `ToUTCTimestamp` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `to_utc_timestamp` expression. /// /// @@ -102,13 +108,16 @@ impl ScalarUDFImpl for SparkToUtcTimestamp { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); + let data_type = args.arg_fields[0].data_type().clone(); + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) + } - Ok(Arc::new(Field::new( - self.name(), - args.arg_fields[0].data_type().clone(), - nullable, - ))) + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { diff --git a/crates/spark/src/function/datetime/unix.rs b/crates/spark/src/function/datetime/unix.rs index 2a6834d1..e5b8c1e7 100644 --- a/crates/spark/src/function/datetime/unix.rs +++ b/crates/spark/src/function/datetime/unix.rs @@ -4,6 +4,7 @@ use std::sync::{Arc, LazyLock}; +use crate::function::nullability::SparkNullability; use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, TimeUnit}; use datafusion_common::cast::as_int64_array; use datafusion_common::types::logical_date; @@ -17,6 +18,9 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; +/// Spark's `UnixDate` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `unix_date` expression. /// /// @@ -74,8 +78,7 @@ impl ScalarUDFImpl for SparkUnixDate { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields[0].is_nullable(); - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } fn documentation(&self) -> Option<&Documentation> { @@ -91,6 +94,10 @@ impl ScalarUDFImpl for SparkUnixDate { args: Vec, info: &SimplifyContext, ) -> Result { + let args = match NULLABILITY.simplify(self, args, info)? { + ExprSimplifyResult::Original(args) => args, + simplified => return Ok(simplified), + }; let [date] = take_function_args(self.name(), args)?; Ok(ExprSimplifyResult::Simplified( date.cast_to(&DataType::Date32, info.schema())? diff --git a/crates/spark/src/function/datetime/weekday.rs b/crates/spark/src/function/datetime/weekday.rs index fe2aa68c..7f563d5e 100644 --- a/crates/spark/src/function/datetime/weekday.rs +++ b/crates/spark/src/function/datetime/weekday.rs @@ -2,20 +2,24 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; - +use crate::function::nullability::SparkNullability; use arrow::array::AsArray; use arrow::compute::{DatePart, date_part}; -use arrow::datatypes::{DataType, Field, FieldRef, Int32Type}; +use arrow::datatypes::{DataType, FieldRef, Int32Type}; use datafusion_common::types::{NativeType, logical_date}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; +/// Spark's `WeekDay` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `weekday` expression. /// /// @@ -79,8 +83,15 @@ impl ScalarUDFImpl for SparkWeekDay { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -113,6 +124,8 @@ impl ScalarUDFImpl for SparkWeekDay { mod tests { use super::*; use arrow::array::{Date32Array, Int32Array}; + use arrow::datatypes::Field; + use std::sync::Arc; #[test] fn test_weekday_return_field_nullability_matches_input() { diff --git a/crates/spark/src/function/hash/sha2.rs b/crates/spark/src/function/hash/sha2.rs index 8dc122cd..fded0f4d 100644 --- a/crates/spark/src/function/hash/sha2.rs +++ b/crates/spark/src/function/hash/sha2.rs @@ -20,6 +20,15 @@ use sha2::{self, Digest}; use std::sync::Arc; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; +use crate::function::nullability::SparkNullability; +use arrow::datatypes::FieldRef; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; + +/// Spark's `Sha2` is null-intolerant and always nullable, because an unsupported bit length +/// returns NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); /// Spark-compatible `sha2` expression. /// @@ -90,6 +99,18 @@ impl ScalarUDFImpl for SparkSha2 { Ok(DataType::Utf8) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/map/map_from_arrays.rs b/crates/spark/src/function/map/map_from_arrays.rs index 971f77f8..eb4b84d8 100644 --- a/crates/spark/src/function/map/map_from_arrays.rs +++ b/crates/spark/src/function/map/map_from_arrays.rs @@ -7,17 +7,22 @@ use crate::function::map::utils::{ get_element_type, get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, map_type_from_key_value_types, }; +use crate::function::nullability::SparkNullability; use arrow::array::{Array, ArrayRef, NullArray}; use arrow::compute::kernels::cast; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_functions::utils::make_scalar_function; -use std::sync::Arc; + +/// Spark's `MapFromArrays` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `map_from_arrays` expression /// @@ -64,8 +69,15 @@ impl ScalarUDFImpl for MapFromArrays { ); // Spark marks map_from_arrays as null intolerant, so the output is // nullable if either input is nullable. - let nullable = keys_field.is_nullable() || values_field.is_nullable(); - Ok(Arc::new(Field::new(self.name(), map_type, nullable))) + Ok(NULLABILITY.return_field(self.name(), map_type, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { @@ -104,6 +116,8 @@ fn map_from_arrays_inner(args: &[ArrayRef], last_value_wins: bool) -> Result #[derive(Debug, PartialEq, Eq, Hash)] @@ -83,11 +90,20 @@ impl ScalarUDFImpl for MapFromEntries { }?; let map_type = map_type_from_key_value_types(keys_type, values_type); - let nullable = entries_field.is_nullable() || entries_element_field.is_nullable(); + let nullable = + NULLABILITY.nullable(args.arg_fields) || entries_element_field.is_nullable(); Ok(Arc::new(Field::new(self.name(), map_type, nullable))) } + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let last_value_wins = map_last_value_wins(&args.config_options); make_scalar_function( diff --git a/crates/spark/src/function/map/str_to_map.rs b/crates/spark/src/function/map/str_to_map.rs index 039041c4..932adc16 100644 --- a/crates/spark/src/function/map/str_to_map.rs +++ b/crates/spark/src/function/map/str_to_map.rs @@ -9,7 +9,7 @@ use arrow::array::{ Array, ArrayRef, MapBuilder, MapFieldNames, StringArrayType, StringBuilder, }; use arrow::buffer::NullBuffer; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::{ as_large_string_array, as_string_array, as_string_view_array, }; @@ -21,10 +21,16 @@ use datafusion_expr::{ use crate::config::map_last_value_wins; use crate::function::map::utils::map_type_from_key_value_types; +use crate::function::nullability::SparkNullability; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; const DEFAULT_PAIR_DELIM: &str = ","; const DEFAULT_KV_DELIM: &str = ":"; +/// Spark's `StringToMap` is a null-intolerant `TernaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `str_to_map` expression /// /// @@ -83,9 +89,16 @@ impl ScalarUDFImpl for SparkStrToMap { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); let map_type = map_type_from_key_value_types(&DataType::Utf8, &DataType::Utf8); - Ok(Arc::new(Field::new(self.name(), map_type, nullable))) + Ok(NULLABILITY.return_field(self.name(), map_type, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { diff --git a/crates/spark/src/function/math/abs.rs b/crates/spark/src/function/math/abs.rs index 6d81b9eb..d871e9c2 100644 --- a/crates/spark/src/function/math/abs.rs +++ b/crates/spark/src/function/math/abs.rs @@ -3,7 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use arrow::array::*; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use arrow::error::ArrowError; use datafusion_common::{DataFusionError, Result, ScalarValue, internal_err}; use datafusion_expr::{ @@ -17,11 +17,17 @@ use datafusion_macros::user_doc; use std::sync::Arc; use crate::function::error_utils::ARITHMETIC_OVERFLOW_ERROR; +use crate::function::nullability::SparkNullability; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; fn arithmetic_overflow_error() -> ArrowError { ArrowError::ComputeError(ARITHMETIC_OVERFLOW_ERROR.to_string()) } +/// Spark's `Abs` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `abs` expression. /// /// @@ -82,11 +88,16 @@ impl ScalarUDFImpl for SparkAbs { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let input_field = &args.arg_fields[0]; - let out_dt = input_field.data_type().clone(); - let out_nullable = input_field.is_nullable(); + let data_type = args.arg_fields[0].data_type().clone(); + Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) + } - Ok(Arc::new(Field::new(self.name(), out_dt, out_nullable))) + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { @@ -239,6 +250,7 @@ pub fn spark_abs( #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::Field; use arrow::datatypes::i256; macro_rules! eval_array_legacy_mode { diff --git a/crates/spark/src/function/math/atan2.rs b/crates/spark/src/function/math/atan2.rs index 38227ee8..645b8502 100644 --- a/crates/spark/src/function/math/atan2.rs +++ b/crates/spark/src/function/math/atan2.rs @@ -4,11 +4,16 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, AsArray, Float64Array}; use arrow::compute::kernels::arity::binary; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Float64Type}; use datafusion_common::Result; use datafusion_common::utils::take_function_args; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -16,6 +21,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `Atan2` is a null-intolerant `BinaryMathExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `atan2` expression. /// /// @@ -80,6 +88,18 @@ impl ScalarUDFImpl for SparkAtan2 { Ok(DataType::Float64) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/bin.rs b/crates/spark/src/function/math/bin.rs index 5c2962aa..ce1b770f 100644 --- a/crates/spark/src/function/math/bin.rs +++ b/crates/spark/src/function/math/bin.rs @@ -2,11 +2,14 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::{Array, ArrayRef, AsArray, StringBuilder}; -use arrow::datatypes::{DataType, Field, FieldRef, Int64Type}; +use arrow::datatypes::{DataType, FieldRef, Int64Type}; use datafusion_common::types::{NativeType, logical_int64}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -15,6 +18,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Bin` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `bin` expression. /// /// @@ -81,11 +87,15 @@ impl ScalarUDFImpl for SparkBin { &self, args: datafusion_expr::ReturnFieldArgs, ) -> Result { - Ok(Arc::new(Field::new( - self.name(), - DataType::Utf8, - args.arg_fields[0].is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/math/ceil.rs b/crates/spark/src/function/math/ceil.rs index 6d08e5eb..477a4c41 100644 --- a/crates/spark/src/function/math/ceil.rs +++ b/crates/spark/src/function/math/ceil.rs @@ -4,16 +4,24 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ArrowNativeTypeOp, AsArray, Decimal128Array}; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Decimal128Type, Float32Type, Float64Type, Int64Type}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_macros::user_doc; +/// Spark's `Ceil` is a null-intolerant `UnaryMathExpression`, which is always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `ceil` expression. /// /// @@ -90,6 +98,18 @@ impl ScalarUDFImpl for SparkCeil { } } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn aliases(&self) -> &[String] { &self.aliases } diff --git a/crates/spark/src/function/math/divide.rs b/crates/spark/src/function/math/divide.rs index 4b9153dc..c37c7630 100644 --- a/crates/spark/src/function/math/divide.rs +++ b/crates/spark/src/function/math/divide.rs @@ -2,13 +2,18 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, Scalar, new_null_array}; use arrow::compute::kernels::boolean::{and, is_not_null}; use arrow::compute::kernels::numeric::div; use arrow::compute::kernels::zip::zip; use arrow::datatypes::DataType; +use arrow::datatypes::FieldRef; use arrow::error::ArrowError; use datafusion_common::{Result, assert_eq_or_internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; @@ -54,6 +59,10 @@ fn spark_divide(args: &[ColumnarValue], enable_ansi_mode: bool) -> Result Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { assert_eq_or_internal_err!( arg_types.len(), diff --git a/crates/spark/src/function/math/expm1.rs b/crates/spark/src/function/math/expm1.rs index b0c280df..43fc4768 100644 --- a/crates/spark/src/function/math/expm1.rs +++ b/crates/spark/src/function/math/expm1.rs @@ -3,10 +3,15 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::error_utils::unsupported_data_type_exec_err; +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, AsArray}; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Float64Type}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -14,6 +19,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Expm1` is a null-intolerant `UnaryMathExpression`, which is always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `expm1` expression. /// /// @@ -66,6 +74,18 @@ impl ScalarUDFImpl for SparkExpm1 { Ok(DataType::Float64) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/factorial.rs b/crates/spark/src/function/math/factorial.rs index 7070e0f5..d1626b3a 100644 --- a/crates/spark/src/function/math/factorial.rs +++ b/crates/spark/src/function/math/factorial.rs @@ -4,21 +4,30 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{Array, Int64Array}; use arrow::datatypes::DataType; use arrow::datatypes::DataType::{Int32, Int64}; +use arrow::datatypes::FieldRef; use datafusion_common::cast::as_int32_array; use datafusion_common::types::logical_int32; use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_err, utils::take_function_args, }; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; use datafusion_expr::Signature; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; +/// Spark's `Factorial` is null-intolerant and always nullable, because a value outside 0 to +/// 20 returns NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `factorial` expression. /// /// @@ -81,6 +90,18 @@ impl ScalarUDFImpl for SparkFactorial { Ok(Int64) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_factorial(&args.args) } diff --git a/crates/spark/src/function/math/floor.rs b/crates/spark/src/function/math/floor.rs index 44b4ab43..1d6b33dd 100644 --- a/crates/spark/src/function/math/floor.rs +++ b/crates/spark/src/function/math/floor.rs @@ -2,12 +2,15 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::cast::AsArray; use arrow::array::types::Decimal128Type; use arrow::array::{ArrowNativeTypeOp, Decimal128Array, Int64Array}; use arrow::compute::kernels::arity::unary; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{DataFusionError, ScalarValue, exec_err, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -15,6 +18,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Floor` is a null-intolerant `UnaryMathExpression`, which is always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `floor` expression. /// /// @@ -79,7 +85,6 @@ impl ScalarUDFImpl for SparkFloor { &self, args: ReturnFieldArgs, ) -> datafusion_common::Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); let return_type = match args.arg_fields[0].data_type() { DataType::Decimal128(p, s) if *s > 0 => { let new_p = (*p - *s as u8 + 1).clamp(1, 38); @@ -97,7 +102,15 @@ impl ScalarUDFImpl for SparkFloor { args.arg_fields[0].data_type() )?, }; - Ok(Arc::new(Field::new(self.name(), return_type, nullable))) + Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> datafusion_common::Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/math/hex.rs b/crates/spark/src/function/math/hex.rs index 07dacdfa..3b23363a 100644 --- a/crates/spark/src/function/math/hex.rs +++ b/crates/spark/src/function/math/hex.rs @@ -5,9 +5,11 @@ use std::str::from_utf8_unchecked; use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{Array, ArrayAccessor, ArrayRef, StringArray, StringBuilder}; use arrow::buffer::{Buffer, OffsetBuffer}; use arrow::datatypes::DataType; +use arrow::datatypes::FieldRef; use arrow::{ array::{as_dictionary_array, as_largestring_array, as_string_array}, datatypes::Int32Type, @@ -22,12 +24,18 @@ use datafusion_common::{ cast::{as_binary_array, as_fixed_size_binary_array, as_int64_array}, exec_datafusion_err, exec_err, }; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; +/// Spark's `Hex` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `hex` expression. /// /// @@ -110,6 +118,21 @@ impl ScalarUDFImpl for SparkHex { }) } + fn return_field_from_args( + &self, + args: ReturnFieldArgs, + ) -> datafusion_common::Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> datafusion_common::Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args( &self, args: ScalarFunctionArgs, diff --git a/crates/spark/src/function/math/hypot.rs b/crates/spark/src/function/math/hypot.rs index 52f26248..3a22a2fd 100644 --- a/crates/spark/src/function/math/hypot.rs +++ b/crates/spark/src/function/math/hypot.rs @@ -4,11 +4,16 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, AsArray, Float64Array}; use arrow::compute::kernels::arity::binary; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Float64Type}; use datafusion_common::Result; use datafusion_common::utils::take_function_args; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -16,6 +21,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `Hypot` is a null-intolerant `BinaryMathExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `hypot` expression. /// /// @@ -79,6 +87,18 @@ impl ScalarUDFImpl for SparkHypot { Ok(DataType::Float64) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/modulus.rs b/crates/spark/src/function/math/modulus.rs index 31c49a40..1fd1ae26 100644 --- a/crates/spark/src/function/math/modulus.rs +++ b/crates/spark/src/function/math/modulus.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, BooleanArray, Scalar, new_null_array}; use arrow::compute::kernels::cast::{CastOptions, cast, cast_with_options}; use arrow::compute::kernels::numeric::add_wrapping; @@ -13,9 +14,13 @@ use arrow::compute::kernels::{ numeric::{neg, rem}, zip::zip, }; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DECIMAL128_MAX_PRECISION, DataType}; use arrow::error::ArrowError; use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err, exec_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, binary::decimal_coercion, @@ -192,6 +197,10 @@ pub fn spark_pmod( Ok(ColumnarValue::Array(result)) } +/// Spark's `Remainder` and `Pmod` are null-intolerant and always nullable, because a zero +/// divisor returns NULL when ANSI mode is off. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// SparkMod implements the Spark-compatible modulo function #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkMod { @@ -233,6 +242,18 @@ impl ScalarUDFImpl for SparkMod { Ok(arg_types[0].clone()) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_mod(&args.args, args.config_options.execution.enable_ansi_mode) } @@ -298,6 +319,18 @@ impl ScalarUDFImpl for SparkPmod { } } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_pmod( &args.args, diff --git a/crates/spark/src/function/math/negative.rs b/crates/spark/src/function/math/negative.rs index 3ad5c401..88ee3998 100644 --- a/crates/spark/src/function/math/negative.rs +++ b/crates/spark/src/function/math/negative.rs @@ -2,12 +2,17 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::types::*; use arrow::array::*; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit}; use bigdecimal::num_traits::WrappingNeg; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -15,6 +20,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `UnaryMinus` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `negative` expression. /// /// @@ -87,6 +95,18 @@ impl ScalarUDFImpl for SparkNegative { Ok(arg_types[0].clone()) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/pow.rs b/crates/spark/src/function/math/pow.rs index 7716ae01..6432c76d 100644 --- a/crates/spark/src/function/math/pow.rs +++ b/crates/spark/src/function/math/pow.rs @@ -14,13 +14,21 @@ use arrow::array::{Array, ArrayRef, Float64Array}; use arrow::compute::binary; use arrow::datatypes::DataType; +use crate::function::nullability::SparkNullability; +use arrow::datatypes::FieldRef; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, }; use datafusion_functions::math::power::PowerFunc; +/// Spark's `Pow` is a null-intolerant `BinaryMathExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible implementation of `pow` / `power`. /// /// Behavioural differences from the DataFusion default: @@ -70,6 +78,18 @@ impl ScalarUDFImpl for SparkPow { self.inner.return_type(arg_types) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { // Only Float64 × Float64 needs the Spark override. // Decimal / integer / mixed-type paths are delegated to the standard diff --git a/crates/spark/src/function/math/rint.rs b/crates/spark/src/function/math/rint.rs index c925dd2c..513a8617 100644 --- a/crates/spark/src/function/math/rint.rs +++ b/crates/spark/src/function/math/rint.rs @@ -4,13 +4,18 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{Array, ArrayRef, AsArray}; use arrow::compute::cast; use arrow::datatypes::DataType::{ Float32, Float64, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, }; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Float32Type, Float64Type}; use datafusion_common::{Result, assert_eq_or_internal_err, exec_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -19,6 +24,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +/// Spark's `Rint` is a null-intolerant `UnaryMathExpression`, which is always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `rint` expression. /// /// @@ -71,6 +79,18 @@ impl ScalarUDFImpl for SparkRint { Ok(Float64) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/unhex.rs b/crates/spark/src/function/math/unhex.rs index 41aa6403..ec1e8298 100644 --- a/crates/spark/src/function/math/unhex.rs +++ b/crates/spark/src/function/math/unhex.rs @@ -2,8 +2,10 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::{Array, ArrayRef, BinaryBuilder}; use arrow::datatypes::DataType; +use arrow::datatypes::FieldRef; use datafusion_common::cast::{ as_large_string_array, as_string_array, as_string_view_array, }; @@ -12,6 +14,9 @@ use datafusion_common::utils::{offset_span_len, take_function_args}; use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, }; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -19,6 +24,10 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Unhex` is null-intolerant and always nullable, because an invalid hex string +/// returns NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `unhex` expression. /// /// @@ -77,6 +86,18 @@ impl ScalarUDFImpl for SparkUnhex { Ok(DataType::Binary) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/math/width_bucket.rs b/crates/spark/src/function/math/width_bucket.rs index 4e2ba485..265623c6 100644 --- a/crates/spark/src/function/math/width_bucket.rs +++ b/crates/spark/src/function/math/width_bucket.rs @@ -27,10 +27,19 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::nullability::SparkNullability; use arrow::array::{Int64Array, Int64Builder}; +use arrow::datatypes::FieldRef; use arrow::datatypes::TimeUnit::Microsecond; use datafusion_expr::Coercion; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; use datafusion_expr::Volatility::Immutable; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; + +/// Spark's `WidthBucket` is null-intolerant and always nullable, because invalid bounds or +/// bucket counts return NULL. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); /// Spark-compatible `width_bucket` expression. /// @@ -155,6 +164,18 @@ impl ScalarUDFImpl for SparkWidthBucket { Ok(Int64) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/nullability.rs b/crates/spark/src/function/nullability.rs index ea1d094b..da12a0b5 100644 --- a/crates/spark/src/function/nullability.rs +++ b/crates/spark/src/function/nullability.rs @@ -39,18 +39,12 @@ use datafusion_expr::{Expr, ExprSchemable, ReturnFieldArgs, ScalarUDFImpl, lit}; /// When a function's result can be NULL: Spark's `Expression.nullable`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[allow( - dead_code, - reason = "declared for the functions that move onto SparkNullability next (#383)" -)] pub(crate) enum SparkNullable { /// When any argument can be NULL: `children.exists(_.nullable)`, the default of /// `UnaryExpression`, `BinaryExpression` and `TernaryExpression`. Children, /// Always, because the function can return NULL for non-NULL arguments. Always, - /// Never. - Never, } /// A function's nullability: Spark's `nullable` and `nullIntolerant`. @@ -68,6 +62,12 @@ impl SparkNullability { Self::new(SparkNullable::Children, true) } + /// A null-intolerant function that can return NULL for non-NULL arguments, so its result + /// is always nullable. Spark overrides `nullable` to `true` for these. + pub(crate) const fn null_intolerant_always_nullable() -> Self { + Self::new(SparkNullable::Always, true) + } + /// Declares `nullable` and `nullIntolerant` separately. pub(crate) const fn new(nullable: SparkNullable, null_intolerant: bool) -> Self { Self { @@ -81,7 +81,6 @@ impl SparkNullability { match self.nullable { SparkNullable::Children => arg_fields.iter().any(|f| f.is_nullable()), SparkNullable::Always => true, - SparkNullable::Never => false, } } @@ -95,6 +94,22 @@ impl SparkNullability { Arc::new(Field::new(name, data_type, self.nullable(arg_fields))) } + /// The return field of a function that computes its type in `ScalarUDFImpl::return_type`, + /// for `ScalarUDFImpl::return_field_from_args`. + pub(crate) fn return_field_from_type( + &self, + func: &F, + args: &ReturnFieldArgs, + ) -> Result { + let arg_types = args + .arg_fields + .iter() + .map(|f| f.data_type().clone()) + .collect::>(); + let data_type = func.return_type(&arg_types)?; + Ok(self.return_field(func.name(), data_type, args.arg_fields)) + } + /// For `ScalarUDFImpl::simplify`: replaces a call to a null-intolerant function that has /// a NULL literal argument with a NULL literal of its return type, as Spark's /// `NullPropagation` does. Otherwise returns the arguments unchanged. @@ -124,8 +139,11 @@ impl SparkNullability { arg_fields: &arg_fields, scalar_arguments: &scalar_arguments, })?; - let null = ScalarValue::try_from(return_field.data_type())?; - Ok(ExprSimplifyResult::Simplified(lit(null))) + match ScalarValue::try_from(return_field.data_type()) { + Ok(null) => Ok(ExprSimplifyResult::Simplified(lit(null))), + // A type that has no NULL `ScalarValue` is left to `invoke_with_args` + Err(_) => Ok(ExprSimplifyResult::Original(args)), + } } } @@ -159,11 +177,10 @@ mod tests { } #[test] - fn test_nullable_always_and_never() { - let always = SparkNullability::new(SparkNullable::Always, true); + fn test_nullable_always() { + let always = SparkNullability::null_intolerant_always_nullable(); + assert!(always.nullable(&[])); assert!(always.nullable(&fields(&[false]))); - let never = SparkNullability::new(SparkNullable::Never, false); - assert!(!never.nullable(&fields(&[true]))); } #[test] diff --git a/crates/spark/src/function/string/base64.rs b/crates/spark/src/function/string/base64.rs index 794eadef..707a3c5d 100644 --- a/crates/spark/src/function/string/base64.rs +++ b/crates/spark/src/function/string/base64.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{Array, ArrayRef, GenericBinaryBuilder, OffsetSizeTrait}; use arrow::datatypes::DataType; use datafusion_common::arrow::datatypes::{Field, FieldRef}; @@ -119,6 +120,9 @@ impl ScalarUDFImpl for SparkBase64 { } } +/// Spark's `UnBase64` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `unbase64` expression. /// /// @@ -191,11 +195,15 @@ impl ScalarUDFImpl for SparkUnBase64 { DataType::LargeBinary => DataType::LargeBinary, _ => DataType::Binary, }; - Ok(Arc::new(Field::new( - self.name(), - return_type, - str.is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { diff --git a/crates/spark/src/function/string/contains.rs b/crates/spark/src/function/string/contains.rs index 19bed03c..f7fae26a 100644 --- a/crates/spark/src/function/string/contains.rs +++ b/crates/spark/src/function/string/contains.rs @@ -4,18 +4,26 @@ use std::sync::Arc; +use crate::function::nullability::SparkNullability; use arrow::array::{Array, ArrayRef, BooleanArray, Scalar}; use arrow::compute::kernels::cast::cast; use arrow::compute::kernels::comparison::contains as arrow_contains; use arrow::datatypes::DataType; +use arrow::datatypes::FieldRef; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_macros::user_doc; +/// Spark's `Contains` is a null-intolerant `StringPredicate`, a `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `contains` expression. /// /// @@ -156,6 +164,18 @@ impl ScalarUDFImpl for SparkContains { Ok(DataType::Boolean) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/crates/spark/src/function/string/is_valid_utf8.rs b/crates/spark/src/function/string/is_valid_utf8.rs index 7c93e9bb..5379948c 100644 --- a/crates/spark/src/function/string/is_valid_utf8.rs +++ b/crates/spark/src/function/string/is_valid_utf8.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, @@ -18,8 +18,14 @@ use datafusion_common::cast::{ use datafusion_common::utils::take_function_args; use datafusion_functions::utils::make_scalar_function; +use crate::function::nullability::SparkNullability; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use std::sync::Arc; +/// Spark's `IsValidUTF8` is null-intolerant and always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `is_valid_utf8` expression. /// /// @@ -86,8 +92,16 @@ impl ScalarUDFImpl for SparkIsValidUtf8 { internal_err!("return_field_from_args should be used instead") } - fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { - Ok(Arc::new(Field::new(self.name(), DataType::Boolean, true))) + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + Ok(NULLABILITY.return_field(self.name(), DataType::Boolean, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/string/like.rs b/crates/spark/src/function/string/like.rs index 32218503..f296613b 100644 --- a/crates/spark/src/function/string/like.rs +++ b/crates/spark/src/function/string/like.rs @@ -2,11 +2,14 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::ArrayRef; use arrow::compute::like; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::ColumnarValue; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -15,6 +18,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `Like` is a null-intolerant `BinaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `like` expression. /// /// @@ -80,12 +86,15 @@ impl ScalarUDFImpl for SparkLike { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - Ok(Arc::new(Field::new( - self.name(), - DataType::Boolean, - nullable, - ))) + Ok(NULLABILITY.return_field(self.name(), DataType::Boolean, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { @@ -109,6 +118,7 @@ mod tests { use crate::function::utils::test::test_scalar_function; use arrow::array::{Array, BooleanArray}; use arrow::datatypes::DataType::Boolean; + use arrow::datatypes::Field; use datafusion_common::ScalarValue; macro_rules! test_like_string_invoke { diff --git a/crates/spark/src/function/string/make_valid_utf8.rs b/crates/spark/src/function/string/make_valid_utf8.rs index fbf75063..a605f28e 100644 --- a/crates/spark/src/function/string/make_valid_utf8.rs +++ b/crates/spark/src/function/string/make_valid_utf8.rs @@ -2,13 +2,16 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, LargeStringArray, StringArray}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_large_binary_array, }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Expr; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -17,6 +20,9 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `MakeValidUTF8` is null-intolerant and always nullable. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable(); + /// Spark-compatible `make_valid_utf8` expression. /// /// @@ -93,11 +99,15 @@ impl ScalarUDFImpl for SparkMakeValidUtf8 { DataType::LargeBinary => Ok(DataType::LargeUtf8), data_type => internal_err!("make_valid_utf8 does not support: {data_type}"), }?; - Ok(Arc::new(Field::new( - self.name(), - return_type, - make_valid_utf8.is_nullable(), - ))) + Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) } fn documentation(&self) -> Option<&Documentation> { diff --git a/crates/spark/src/function/string/soundex.rs b/crates/spark/src/function/string/soundex.rs index 2b533e80..50c15d0f 100644 --- a/crates/spark/src/function/string/soundex.rs +++ b/crates/spark/src/function/string/soundex.rs @@ -2,17 +2,25 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait}; use arrow::datatypes::DataType; +use arrow::datatypes::FieldRef; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ColumnarValue, Documentation, Signature, Volatility}; use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl}; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `SoundEx` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `soundex` expression. /// /// @@ -72,6 +80,18 @@ impl ScalarUDFImpl for SparkSoundex { } } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_soundex_inner, vec![])(&args.args) } diff --git a/crates/spark/src/function/string/space.rs b/crates/spark/src/function/string/space.rs index dea5e05e..dc026f46 100644 --- a/crates/spark/src/function/string/space.rs +++ b/crates/spark/src/function/string/space.rs @@ -2,13 +2,18 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::nullability::SparkNullability; use arrow::array::{ Array, ArrayRef, DictionaryArray, Int32Array, StringArray, StringBuilder, as_dictionary_array, }; +use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Int32Type}; use datafusion_common::cast::as_int32_array; use datafusion_common::{Result, ScalarValue, exec_err}; +use datafusion_expr::Expr; +use datafusion_expr::ReturnFieldArgs; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -16,6 +21,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark's `StringSpace` is a null-intolerant `UnaryExpression`. +const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); + /// Spark-compatible `space` expression. /// /// @@ -86,6 +94,18 @@ impl ScalarUDFImpl for SparkSpace { Ok(return_type) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + NULLABILITY.return_field_from_type(self, &args) + } + + fn simplify( + &self, + args: Vec, + info: &SimplifyContext, + ) -> Result { + NULLABILITY.simplify(self, args, info) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_space(&args.args) } diff --git a/crates/sqllogictest/test_files/datetime/make_dt_interval.slt b/crates/sqllogictest/test_files/datetime/make_dt_interval.slt index be19b359..2a77ad0a 100644 --- a/crates/sqllogictest/test_files/datetime/make_dt_interval.slt +++ b/crates/sqllogictest/test_files/datetime/make_dt_interval.slt @@ -143,9 +143,6 @@ SELECT make_dt_interval(1491308, 0, 0, 0); ---- 1491308 days 0 hours 0 mins 0.000000 secs -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(1491309, 0, 0, 0); ---- @@ -167,9 +164,6 @@ SELECT make_dt_interval(106751991, 0, 0, 0); 106751991 days 0 hours 0 mins 0.000000 secs # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(106751992, 0, 0, 0); ---- @@ -181,9 +175,6 @@ SELECT make_dt_interval(-106751991, 0, 0, 0); -106751991 days 0 hours 0 mins 0.000000 secs # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(-106751992, 0, 0, 0); ---- @@ -191,18 +182,12 @@ NULL # Mixed-sign cancellation checks (Spark raises INTERVAL_ARITHMETIC_OVERFLOW, returns NULL pending #20) # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(106751992, -24, 0, 0); ---- NULL # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(106751991, 5, -60, 0); ---- @@ -221,18 +206,12 @@ SELECT make_dt_interval(-106751991, -4, 0, -54.775808); # One microsecond beyond boundaries returns NULL (overflow tracked in #20) # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(106751991, 4, 0, 54.775808); ---- NULL # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) -# Without constant folding this fails because the result is declared non-nullable -# (https://github.com/apache/datafusion-spark/issues/315). -skipif evaluate query ? SELECT make_dt_interval(-106751991, -4, 0, -54.775809); ---- diff --git a/crates/sqllogictest/test_files/math/atan2.slt b/crates/sqllogictest/test_files/math/atan2.slt index 4b837a5b..6ad33337 100644 --- a/crates/sqllogictest/test_files/math/atan2.slt +++ b/crates/sqllogictest/test_files/math/atan2.slt @@ -166,3 +166,11 @@ SELECT atan2(a, b) FROM (VALUES (0.0::double, 1.0::double), (1.0::double, 1.0::d 0.7853981633974483 NULL 3.141592653589793 + +# atan2 is null-intolerant, as in Spark: a NULL argument makes every row NULL +query RR +SELECT atan2(column1, NULL::double), atan2(NULL::double, column1) +FROM VALUES (1.0::double), (NULL::double); +---- +NULL NULL +NULL NULL diff --git a/crates/sqllogictest/test_files/math/pmod.slt b/crates/sqllogictest/test_files/math/pmod.slt index 7a7b4e63..9efcda1e 100644 --- a/crates/sqllogictest/test_files/math/pmod.slt +++ b/crates/sqllogictest/test_files/math/pmod.slt @@ -560,3 +560,11 @@ query R SELECT pmod(-7.2, 3.0) as pmod_scalar_4; ---- 1.7999999999999998 + +# pmod is null-intolerant, as in Spark: a NULL argument makes every row NULL +query II +SELECT pmod(column1, NULL::int), pmod(NULL::int, column1) +FROM VALUES (7::int), (NULL::int); +---- +NULL NULL +NULL NULL diff --git a/crates/sqllogictest/test_files/predicate/like.slt b/crates/sqllogictest/test_files/predicate/like.slt index 57f71e9a..297bf65e 100644 --- a/crates/sqllogictest/test_files/predicate/like.slt +++ b/crates/sqllogictest/test_files/predicate/like.slt @@ -69,3 +69,11 @@ query B SELECT like('Spark'::string, NULL::string); ---- NULL + +# like is null-intolerant, as in Spark: a NULL argument makes every row NULL +query BB +SELECT column1 LIKE NULL::string, like(column1, NULL::string) +FROM VALUES ('Spark SQL'::string), (NULL::string); +---- +NULL NULL +NULL NULL diff --git a/crates/sqllogictest/test_files/string/contains.slt b/crates/sqllogictest/test_files/string/contains.slt index ade7a622..02266ae9 100644 --- a/crates/sqllogictest/test_files/string/contains.slt +++ b/crates/sqllogictest/test_files/string/contains.slt @@ -175,3 +175,11 @@ true NULL NULL false + +# contains is null-intolerant, as in Spark: a NULL argument makes every row NULL +query BB +SELECT contains(column1, NULL::string), contains(NULL::string, column1) +FROM VALUES ('Spark SQL'::string), (NULL::string); +---- +NULL NULL +NULL NULL From 6bc141dfb72d8dcb36a12c36d9b499d9a5272208 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 07:29:37 -0700 Subject: [PATCH 3/5] refactor(spark): implement SparkNullability's ScalarUDFImpl methods with a macro Add spark_nullability!, which implements simplify, and with return_type also return_field_from_args, from a function's SparkNullability. This replaces the same eight-line simplify, and the three-line return_field_from_args, in each function. Part of #383. --- .../src/function/array/array_contains.rs | 12 +---- .../spark/src/function/array/array_extrema.rs | 29 ++--------- .../src/function/array/array_position.rs | 18 +------ .../spark/src/function/array/array_remove.rs | 18 +------ crates/spark/src/function/array/slice.rs | 12 +---- .../spark/src/function/bitwise/bit_count.rs | 12 +---- crates/spark/src/function/bitwise/bit_get.rs | 12 +---- .../spark/src/function/bitwise/bit_shift.rs | 12 +---- .../spark/src/function/bitwise/bitwise_not.rs | 12 +---- .../spark/src/function/datetime/date_add.rs | 12 +---- .../function/datetime/date_from_unix_date.rs | 12 +---- .../spark/src/function/datetime/date_sub.rs | 12 +---- crates/spark/src/function/datetime/dayname.rs | 12 +---- crates/spark/src/function/datetime/extract.rs | 42 ++-------------- .../function/datetime/from_utc_timestamp.rs | 12 +---- .../spark/src/function/datetime/last_day.rs | 12 +---- .../spark/src/function/datetime/make_date.rs | 12 +---- .../src/function/datetime/make_dt_interval.rs | 12 +---- .../src/function/datetime/make_interval.rs | 18 +------ .../spark/src/function/datetime/monthname.rs | 12 +---- .../spark/src/function/datetime/next_day.rs | 12 +---- .../src/function/datetime/to_utc_timestamp.rs | 12 +---- crates/spark/src/function/datetime/weekday.rs | 12 +---- crates/spark/src/function/hash/crc32.rs | 12 +---- crates/spark/src/function/hash/sha1.rs | 12 +---- crates/spark/src/function/hash/sha2.rs | 18 +------ .../spark/src/function/map/map_from_arrays.rs | 12 +---- .../src/function/map/map_from_entries.rs | 12 +---- crates/spark/src/function/map/str_to_map.rs | 12 +---- crates/spark/src/function/math/abs.rs | 12 +---- crates/spark/src/function/math/atan2.rs | 18 +------ crates/spark/src/function/math/bin.rs | 12 +---- crates/spark/src/function/math/ceil.rs | 18 +------ crates/spark/src/function/math/divide.rs | 18 +------ crates/spark/src/function/math/expm1.rs | 18 +------ crates/spark/src/function/math/factorial.rs | 18 +------ crates/spark/src/function/math/floor.rs | 12 +---- crates/spark/src/function/math/hex.rs | 21 +------- crates/spark/src/function/math/hypot.rs | 18 +------ crates/spark/src/function/math/modulus.rs | 30 ++---------- crates/spark/src/function/math/negative.rs | 18 +------ crates/spark/src/function/math/pow.rs | 18 +------ crates/spark/src/function/math/rint.rs | 18 +------ crates/spark/src/function/math/unhex.rs | 18 +------ .../spark/src/function/math/width_bucket.rs | 18 +------ crates/spark/src/function/nullability.rs | 48 ++++++++++++++++--- crates/spark/src/function/string/ascii.rs | 12 +---- crates/spark/src/function/string/base64.rs | 10 +--- crates/spark/src/function/string/char.rs | 12 +---- crates/spark/src/function/string/contains.rs | 18 +------ .../src/function/string/is_valid_utf8.rs | 12 +---- crates/spark/src/function/string/length.rs | 12 +---- crates/spark/src/function/string/like.rs | 12 +---- .../src/function/string/make_valid_utf8.rs | 12 +---- crates/spark/src/function/string/soundex.rs | 18 +------ crates/spark/src/function/string/space.rs | 18 +------ crates/spark/src/function/string/substring.rs | 12 +---- 57 files changed, 158 insertions(+), 742 deletions(-) diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 7f1e9886..502c6c6b 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -3,16 +3,14 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_floats; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; use arrow::buffer::{BooleanBuffer, NullBuffer}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{Result, exec_err}; -use datafusion_expr::Expr; use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; @@ -82,13 +80,7 @@ impl ScalarUDFImpl for SparkArrayContains { ))) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, mut args: ScalarFunctionArgs) -> Result { for arg in &mut args.args { diff --git a/crates/spark/src/function/array/array_extrema.rs b/crates/spark/src/function/array/array_extrema.rs index fa9d03d4..00d313c1 100644 --- a/crates/spark/src/function/array/array_extrema.rs +++ b/crates/spark/src/function/array/array_extrema.rs @@ -5,7 +5,7 @@ //! Spark-compatible `array_max` and `array_min`. use crate::function::float_normalize::normalize_array; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use crate::function::type_coercion::{is_orderable, spark_sql_type}; use arrow::array::{ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray, OffsetSizeTrait, @@ -19,9 +19,6 @@ use arrow::datatypes::{ }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -90,17 +87,7 @@ impl ScalarUDFImpl for SparkArrayMax { element_type(self.name(), arg_types) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn coerce_types(&self, arg_types: &[DataType]) -> Result> { coerce_array(self.name(), arg_types) @@ -171,17 +158,7 @@ impl ScalarUDFImpl for SparkArrayMin { element_type(self.name(), arg_types) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn coerce_types(&self, arg_types: &[DataType]) -> Result> { coerce_array(self.name(), arg_types) diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 113942ab..6c5c5333 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -3,7 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_array; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use crate::function::type_coercion::{ is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; @@ -12,13 +12,9 @@ use arrow::array::{ }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -93,17 +89,7 @@ impl ScalarUDFImpl for SparkArrayPosition { Ok(DataType::Int64) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn coerce_types(&self, arg_types: &[DataType]) -> Result> { let [array, element] = take_function_args(self.name(), arg_types)?; diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs index 04e976c0..125ee163 100644 --- a/crates/spark/src/function/array/array_remove.rs +++ b/crates/spark/src/function/array/array_remove.rs @@ -3,7 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_array; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use crate::function::type_coercion::{ is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; @@ -14,13 +14,9 @@ use arrow::array::{ use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer}; use arrow::compute::kernels::cmp::eq; use arrow::compute::{SortOptions, filter}; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Field, Float32Type, Float64Type}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -93,17 +89,7 @@ impl ScalarUDFImpl for SparkArrayRemove { Ok(array.clone()) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn coerce_types(&self, arg_types: &[DataType]) -> Result> { let [array, element] = take_function_args(self.name(), arg_types)?; diff --git a/crates/spark/src/function/array/slice.rs b/crates/spark/src/function/array/slice.rs index 9cba749d..14234266 100644 --- a/crates/spark/src/function/array/slice.rs +++ b/crates/spark/src/function/array/slice.rs @@ -9,8 +9,6 @@ use datafusion_common::utils::ListCoercion; use datafusion_common::{ Result, ScalarValue, exec_err, internal_err, utils::take_function_args, }; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, @@ -20,7 +18,7 @@ use datafusion_functions_nested::extract::array_slice_udf; use datafusion_macros::user_doc; use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `Slice` is a null-intolerant `TernaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -111,13 +109,7 @@ impl ScalarUDFImpl for SparkSlice { Ok(NULLABILITY.return_field("slice", data_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/bitwise/bit_count.rs b/crates/spark/src/function/bitwise/bit_count.rs index 1f421971..ce2b4ebf 100644 --- a/crates/spark/src/function/bitwise/bit_count.rs +++ b/crates/spark/src/function/bitwise/bit_count.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray, Int32Array}; use arrow::datatypes::{ DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, @@ -12,8 +12,6 @@ use arrow::datatypes::{ }; use datafusion_common::cast::as_boolean_array; use datafusion_common::{Result, internal_err, plan_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -100,13 +98,7 @@ impl ScalarUDFImpl for SparkBitCount { Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/bitwise/bit_get.rs b/crates/spark/src/function/bitwise/bit_get.rs index 6ff69811..d1a15ef5 100644 --- a/crates/spark/src/function/bitwise/bit_get.rs +++ b/crates/spark/src/function/bitwise/bit_get.rs @@ -14,8 +14,6 @@ use arrow::datatypes::{ArrowNativeType, DataType, FieldRef, Int8Type, Int32Type} use datafusion_common::types::{NativeType, logical_int32}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -23,7 +21,7 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `BitwiseGet` is a null-intolerant `BinaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -107,13 +105,7 @@ impl ScalarUDFImpl for SparkBitGet { Ok(NULLABILITY.return_field(self.name(), DataType::Int8, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/bitwise/bit_shift.rs b/crates/spark/src/function/bitwise/bit_shift.rs index 322efdd0..19917afa 100644 --- a/crates/spark/src/function/bitwise/bit_shift.rs +++ b/crates/spark/src/function/bitwise/bit_shift.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, Int32Array, PrimitiveArray}; use arrow::compute; use arrow::datatypes::{ @@ -16,8 +16,6 @@ use datafusion_common::types::{ }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -363,13 +361,7 @@ impl ScalarUDFImpl for SparkBitShift { Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { match self.bit_shift_type { diff --git a/crates/spark/src/function/bitwise/bitwise_not.rs b/crates/spark/src/function/bitwise/bitwise_not.rs index 7e34e262..6b6d141e 100644 --- a/crates/spark/src/function/bitwise/bitwise_not.rs +++ b/crates/spark/src/function/bitwise/bitwise_not.rs @@ -2,13 +2,11 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::*; use arrow::compute::kernels::bitwise; use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type}; use datafusion_common::{Result, internal_err, plan_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ColumnarValue, Documentation, TypeSignature, Volatility}; use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature}; use datafusion_functions::utils::make_scalar_function; @@ -87,13 +85,7 @@ impl ScalarUDFImpl for SparkBitwiseNot { Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/date_add.rs b/crates/spark/src/function/datetime/date_add.rs index 30799b4b..959f2174 100644 --- a/crates/spark/src/function/datetime/date_add.rs +++ b/crates/spark/src/function/datetime/date_add.rs @@ -12,8 +12,6 @@ use datafusion_common::cast::{ }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -21,7 +19,7 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `DateAdd` is a null-intolerant `BinaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -105,13 +103,7 @@ impl ScalarUDFImpl for SparkDateAdd { Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/date_from_unix_date.rs b/crates/spark/src/function/datetime/date_from_unix_date.rs index a7066a44..9871e011 100644 --- a/crates/spark/src/function/datetime/date_from_unix_date.rs +++ b/crates/spark/src/function/datetime/date_from_unix_date.rs @@ -16,9 +16,7 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::int32_coercion; -use crate::function::nullability::SparkNullability; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `DateFromUnixDate` is a null-intolerant `UnaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -90,13 +88,7 @@ impl ScalarUDFImpl for SparkDateFromUnixDate { Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/date_sub.rs b/crates/spark/src/function/datetime/date_sub.rs index 8c025bbd..dc79641f 100644 --- a/crates/spark/src/function/datetime/date_sub.rs +++ b/crates/spark/src/function/datetime/date_sub.rs @@ -11,8 +11,6 @@ use datafusion_common::cast::{ as_date32_array, as_int8_array, as_int16_array, as_int32_array, }; use datafusion_common::{Result, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -20,7 +18,7 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `DateSub` is a null-intolerant `BinaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -97,13 +95,7 @@ impl ScalarUDFImpl for SparkDateSub { Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/dayname.rs b/crates/spark/src/function/datetime/dayname.rs index dc5c1b2b..3842d6ba 100644 --- a/crates/spark/src/function/datetime/dayname.rs +++ b/crates/spark/src/function/datetime/dayname.rs @@ -13,9 +13,7 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::{date_part_name, date_signature}; -use crate::function::nullability::SparkNullability; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use crate::function::nullability::{SparkNullability, spark_nullability}; const DAY_NAMES: [&str; 7] = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]; @@ -91,13 +89,7 @@ impl ScalarUDFImpl for SparkDayName { Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/extract.rs b/crates/spark/src/function/datetime/extract.rs index 658b810c..1a63ba6c 100644 --- a/crates/spark/src/function/datetime/extract.rs +++ b/crates/spark/src/function/datetime/extract.rs @@ -2,16 +2,12 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::ArrayRef; use arrow::compute::{DatePart, date_part}; use arrow::datatypes::DataType; -use arrow::datatypes::FieldRef; use datafusion_common::Result; use datafusion_common::utils::take_function_args; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -86,17 +82,7 @@ impl ScalarUDFImpl for SparkHour { Ok(DataType::Int32) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() @@ -169,17 +155,7 @@ impl ScalarUDFImpl for SparkMinute { Ok(DataType::Int32) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() @@ -253,17 +229,7 @@ impl ScalarUDFImpl for SparkSecond { Ok(DataType::Int32) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/from_utc_timestamp.rs b/crates/spark/src/function/datetime/from_utc_timestamp.rs index 7a32450b..0763130f 100644 --- a/crates/spark/src/function/datetime/from_utc_timestamp.rs +++ b/crates/spark/src/function/datetime/from_utc_timestamp.rs @@ -7,7 +7,7 @@ use std::hash::{Hash, Hasher}; use std::sync::Arc; use super::timezone::TimeZoneRules; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, ArrayRef, AsArray, BooleanArray, PrimitiveBuilder, StringArrayType, new_empty_array, @@ -21,8 +21,6 @@ use arrow::record_batch::RecordBatch; use datafusion_common::types::{NativeType, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_datafusion_err, exec_err, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ExpressionPlacement, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -123,13 +121,7 @@ impl ScalarUDFImpl for SparkFromUtcTimestamp { Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_from_utc_timestamp, vec![])(&args.args) diff --git a/crates/spark/src/function/datetime/last_day.rs b/crates/spark/src/function/datetime/last_day.rs index f705b6d6..b260c5ae 100644 --- a/crates/spark/src/function/datetime/last_day.rs +++ b/crates/spark/src/function/datetime/last_day.rs @@ -4,14 +4,12 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray, Date32Array}; use arrow::datatypes::{DataType, Date32Type, FieldRef}; use chrono::{Datelike, Duration, NaiveDate}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_datafusion_err, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -76,13 +74,7 @@ impl ScalarUDFImpl for SparkLastDay { Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/make_date.rs b/crates/spark/src/function/datetime/make_date.rs index 781c2089..b1bd7e94 100644 --- a/crates/spark/src/function/datetime/make_date.rs +++ b/crates/spark/src/function/datetime/make_date.rs @@ -16,9 +16,7 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::int32_coercion; -use crate::function::nullability::SparkNullability; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use crate::function::nullability::{SparkNullability, spark_nullability}; const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. SQLSTATE: 22003"#; @@ -110,13 +108,7 @@ impl ScalarUDFImpl for SparkMakeDate { Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/make_dt_interval.rs b/crates/spark/src/function/datetime/make_dt_interval.rs index c58d73ca..b740af21 100644 --- a/crates/spark/src/function/datetime/make_dt_interval.rs +++ b/crates/spark/src/function/datetime/make_dt_interval.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, ArrayRef, AsArray, DurationMicrosecondBuilder, PrimitiveArray, }; @@ -14,8 +14,6 @@ use datafusion_common::types::{NativeType, logical_float64, logical_int32}; use datafusion_common::{ DataFusionError, Result, ScalarValue, internal_err, plan_datafusion_err, }; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -145,13 +143,7 @@ impl ScalarUDFImpl for SparkMakeDtInterval { )) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.is_empty() { diff --git a/crates/spark/src/function/datetime/make_interval.rs b/crates/spark/src/function/datetime/make_interval.rs index 80f97322..e262d60a 100644 --- a/crates/spark/src/function/datetime/make_interval.rs +++ b/crates/spark/src/function/datetime/make_interval.rs @@ -4,17 +4,13 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, IntervalMonthDayNanoBuilder, PrimitiveArray}; use arrow::datatypes::DataType::Interval; -use arrow::datatypes::FieldRef; use arrow::datatypes::IntervalUnit::MonthDayNano; use arrow::datatypes::{DataType, IntervalMonthDayNano}; use datafusion_common::types::{NativeType, logical_float64, logical_int32}; use datafusion_common::{DataFusionError, Result, ScalarValue, plan_datafusion_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -167,17 +163,7 @@ impl ScalarUDFImpl for SparkMakeInterval { Ok(Interval(MonthDayNano)) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.is_empty() { diff --git a/crates/spark/src/function/datetime/monthname.rs b/crates/spark/src/function/datetime/monthname.rs index 820ec164..f38bee27 100644 --- a/crates/spark/src/function/datetime/monthname.rs +++ b/crates/spark/src/function/datetime/monthname.rs @@ -13,9 +13,7 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use super::date_part_name::{date_part_name, date_signature}; -use crate::function::nullability::SparkNullability; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use crate::function::nullability::{SparkNullability, spark_nullability}; const MONTH_NAMES: [&str; 12] = [ "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", @@ -92,13 +90,7 @@ impl ScalarUDFImpl for SparkMonthName { Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/next_day.rs b/crates/spark/src/function/datetime/next_day.rs index 0de9dd12..4eb078c5 100644 --- a/crates/spark/src/function/datetime/next_day.rs +++ b/crates/spark/src/function/datetime/next_day.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, ArrayRef, AsArray, Date32Array, Date32Builder, StringArrayType, }; @@ -12,8 +12,6 @@ use arrow::datatypes::{DataType, Date32Type, FieldRef}; use chrono::{Datelike, Weekday}; use datafusion_common::types::{NativeType, logical_date, logical_string}; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -111,13 +109,7 @@ impl ScalarUDFImpl for SparkNextDay { Ok(NULLABILITY.return_field(self.name(), DataType::Date32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/datetime/to_utc_timestamp.rs b/crates/spark/src/function/datetime/to_utc_timestamp.rs index 16f60181..1790a75f 100644 --- a/crates/spark/src/function/datetime/to_utc_timestamp.rs +++ b/crates/spark/src/function/datetime/to_utc_timestamp.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::timezone::Tz; use arrow::array::{Array, ArrayRef, AsArray, PrimitiveBuilder, StringArrayType}; use arrow::datatypes::TimeUnit; @@ -18,8 +18,6 @@ use datafusion_common::utils::take_function_args; use datafusion_common::{ Result, exec_datafusion_err, exec_err, internal_datafusion_err, internal_err, }; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -112,13 +110,7 @@ impl ScalarUDFImpl for SparkToUtcTimestamp { Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(to_utc_timestamp, vec![])(&args.args) diff --git a/crates/spark/src/function/datetime/weekday.rs b/crates/spark/src/function/datetime/weekday.rs index 7f563d5e..74ead9ef 100644 --- a/crates/spark/src/function/datetime/weekday.rs +++ b/crates/spark/src/function/datetime/weekday.rs @@ -2,15 +2,13 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::AsArray; use arrow::compute::{DatePart, date_part}; use arrow::datatypes::{DataType, FieldRef, Int32Type}; use datafusion_common::types::{NativeType, logical_date}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -86,13 +84,7 @@ impl ScalarUDFImpl for SparkWeekDay { Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/hash/crc32.rs b/crates/spark/src/function/hash/crc32.rs index bf3adf75..8cd86385 100644 --- a/crates/spark/src/function/hash/crc32.rs +++ b/crates/spark/src/function/hash/crc32.rs @@ -22,10 +22,8 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `Crc32` is a null-intolerant `UnaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -101,13 +99,7 @@ impl ScalarUDFImpl for SparkCrc32 { Ok(NULLABILITY.return_field(self.name(), DataType::Int64, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/hash/sha1.rs b/crates/spark/src/function/hash/sha1.rs index 20835c60..8164e712 100644 --- a/crates/spark/src/function/hash/sha1.rs +++ b/crates/spark/src/function/hash/sha1.rs @@ -23,10 +23,8 @@ use datafusion_macros::user_doc; use sha1::{Digest, Sha1}; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `Sha1` is a null-intolerant `UnaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -109,13 +107,7 @@ impl ScalarUDFImpl for SparkSha1 { Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/hash/sha2.rs b/crates/spark/src/function/hash/sha2.rs index fded0f4d..6721b2f6 100644 --- a/crates/spark/src/function/hash/sha2.rs +++ b/crates/spark/src/function/hash/sha2.rs @@ -20,11 +20,7 @@ use sha2::{self, Digest}; use std::sync::Arc; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; -use crate::function::nullability::SparkNullability; -use arrow::datatypes::FieldRef; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `Sha2` is null-intolerant and always nullable, because an unsupported bit length /// returns NULL. @@ -99,17 +95,7 @@ impl ScalarUDFImpl for SparkSha2 { Ok(DataType::Utf8) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/map/map_from_arrays.rs b/crates/spark/src/function/map/map_from_arrays.rs index eb4b84d8..2a86831c 100644 --- a/crates/spark/src/function/map/map_from_arrays.rs +++ b/crates/spark/src/function/map/map_from_arrays.rs @@ -7,14 +7,12 @@ use crate::function::map::utils::{ get_element_type, get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, map_type_from_key_value_types, }; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, NullArray}; use arrow::compute::kernels::cast; use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -72,13 +70,7 @@ impl ScalarUDFImpl for MapFromArrays { Ok(NULLABILITY.return_field(self.name(), map_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let last_value_wins = map_last_value_wins(&args.config_options); diff --git a/crates/spark/src/function/map/map_from_entries.rs b/crates/spark/src/function/map/map_from_entries.rs index c8fb54ae..1a4bb6f1 100644 --- a/crates/spark/src/function/map/map_from_entries.rs +++ b/crates/spark/src/function/map/map_from_entries.rs @@ -9,14 +9,12 @@ use crate::function::map::utils::{ get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, map_type_from_key_value_types, }; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, NullBufferBuilder, StructArray}; use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -96,13 +94,7 @@ impl ScalarUDFImpl for MapFromEntries { Ok(Arc::new(Field::new(self.name(), map_type, nullable))) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let last_value_wins = map_last_value_wins(&args.config_options); diff --git a/crates/spark/src/function/map/str_to_map.rs b/crates/spark/src/function/map/str_to_map.rs index 932adc16..dd32858b 100644 --- a/crates/spark/src/function/map/str_to_map.rs +++ b/crates/spark/src/function/map/str_to_map.rs @@ -21,9 +21,7 @@ use datafusion_expr::{ use crate::config::map_last_value_wins; use crate::function::map::utils::map_type_from_key_value_types; -use crate::function::nullability::SparkNullability; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use crate::function::nullability::{SparkNullability, spark_nullability}; const DEFAULT_PAIR_DELIM: &str = ","; const DEFAULT_KV_DELIM: &str = ":"; @@ -93,13 +91,7 @@ impl ScalarUDFImpl for SparkStrToMap { Ok(NULLABILITY.return_field(self.name(), map_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let last_value_wins = map_last_value_wins(&args.config_options); diff --git a/crates/spark/src/function/math/abs.rs b/crates/spark/src/function/math/abs.rs index d871e9c2..879a8779 100644 --- a/crates/spark/src/function/math/abs.rs +++ b/crates/spark/src/function/math/abs.rs @@ -17,9 +17,7 @@ use datafusion_macros::user_doc; use std::sync::Arc; use crate::function::error_utils::ARITHMETIC_OVERFLOW_ERROR; -use crate::function::nullability::SparkNullability; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use crate::function::nullability::{SparkNullability, spark_nullability}; fn arithmetic_overflow_error() -> ArrowError { ArrowError::ComputeError(ARITHMETIC_OVERFLOW_ERROR.to_string()) @@ -92,13 +90,7 @@ impl ScalarUDFImpl for SparkAbs { Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/atan2.rs b/crates/spark/src/function/math/atan2.rs index 645b8502..4d6647be 100644 --- a/crates/spark/src/function/math/atan2.rs +++ b/crates/spark/src/function/math/atan2.rs @@ -4,16 +4,12 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray, Float64Array}; use arrow::compute::kernels::arity::binary; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Float64Type}; use datafusion_common::Result; use datafusion_common::utils::take_function_args; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -88,17 +84,7 @@ impl ScalarUDFImpl for SparkAtan2 { Ok(DataType::Float64) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/bin.rs b/crates/spark/src/function/math/bin.rs index ce1b770f..fa73503e 100644 --- a/crates/spark/src/function/math/bin.rs +++ b/crates/spark/src/function/math/bin.rs @@ -2,14 +2,12 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, AsArray, StringBuilder}; use arrow::datatypes::{DataType, FieldRef, Int64Type}; use datafusion_common::types::{NativeType, logical_int64}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -90,13 +88,7 @@ impl ScalarUDFImpl for SparkBin { Ok(NULLABILITY.return_field(self.name(), DataType::Utf8, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/ceil.rs b/crates/spark/src/function/math/ceil.rs index 477a4c41..ad482b66 100644 --- a/crates/spark/src/function/math/ceil.rs +++ b/crates/spark/src/function/math/ceil.rs @@ -4,15 +4,11 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrowNativeTypeOp, AsArray, Decimal128Array}; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Decimal128Type, Float32Type, Float64Type, Int64Type}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -98,17 +94,7 @@ impl ScalarUDFImpl for SparkCeil { } } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn aliases(&self) -> &[String] { &self.aliases diff --git a/crates/spark/src/function/math/divide.rs b/crates/spark/src/function/math/divide.rs index c37c7630..cada4e5e 100644 --- a/crates/spark/src/function/math/divide.rs +++ b/crates/spark/src/function/math/divide.rs @@ -2,18 +2,14 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, Scalar, new_null_array}; use arrow::compute::kernels::boolean::{and, is_not_null}; use arrow::compute::kernels::numeric::div; use arrow::compute::kernels::zip::zip; use arrow::datatypes::DataType; -use arrow::datatypes::FieldRef; use arrow::error::ArrowError; use datafusion_common::{Result, assert_eq_or_internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; @@ -97,17 +93,7 @@ impl ScalarUDFImpl for SparkDivide { Ok(DataType::Float64) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn coerce_types(&self, arg_types: &[DataType]) -> Result> { assert_eq_or_internal_err!( diff --git a/crates/spark/src/function/math/expm1.rs b/crates/spark/src/function/math/expm1.rs index 43fc4768..e03e2150 100644 --- a/crates/spark/src/function/math/expm1.rs +++ b/crates/spark/src/function/math/expm1.rs @@ -3,15 +3,11 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::error_utils::unsupported_data_type_exec_err; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray}; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Float64Type}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -74,17 +70,7 @@ impl ScalarUDFImpl for SparkExpm1 { Ok(DataType::Float64) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/factorial.rs b/crates/spark/src/function/math/factorial.rs index d1626b3a..ecaac754 100644 --- a/crates/spark/src/function/math/factorial.rs +++ b/crates/spark/src/function/math/factorial.rs @@ -4,20 +4,16 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, Int64Array}; use arrow::datatypes::DataType; use arrow::datatypes::DataType::{Int32, Int64}; -use arrow::datatypes::FieldRef; use datafusion_common::cast::as_int32_array; use datafusion_common::types::logical_int32; use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_err, utils::take_function_args, }; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; use datafusion_expr::Signature; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, TypeSignatureClass, Volatility, @@ -90,17 +86,7 @@ impl ScalarUDFImpl for SparkFactorial { Ok(Int64) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_factorial(&args.args) diff --git a/crates/spark/src/function/math/floor.rs b/crates/spark/src/function/math/floor.rs index 1d6b33dd..18673ca3 100644 --- a/crates/spark/src/function/math/floor.rs +++ b/crates/spark/src/function/math/floor.rs @@ -2,15 +2,13 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::cast::AsArray; use arrow::array::types::Decimal128Type; use arrow::array::{ArrowNativeTypeOp, Decimal128Array, Int64Array}; use arrow::compute::kernels::arity::unary; use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{DataFusionError, ScalarValue, exec_err, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -105,13 +103,7 @@ impl ScalarUDFImpl for SparkFloor { Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> datafusion_common::Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/hex.rs b/crates/spark/src/function/math/hex.rs index 3b23363a..7bdc3aec 100644 --- a/crates/spark/src/function/math/hex.rs +++ b/crates/spark/src/function/math/hex.rs @@ -5,11 +5,10 @@ use std::str::from_utf8_unchecked; use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayAccessor, ArrayRef, StringArray, StringBuilder}; use arrow::buffer::{Buffer, OffsetBuffer}; use arrow::datatypes::DataType; -use arrow::datatypes::FieldRef; use arrow::{ array::{as_dictionary_array, as_largestring_array, as_string_array}, datatypes::Int32Type, @@ -24,9 +23,6 @@ use datafusion_common::{ cast::{as_binary_array, as_fixed_size_binary_array, as_int64_array}, exec_datafusion_err, exec_err, }; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, @@ -118,20 +114,7 @@ impl ScalarUDFImpl for SparkHex { }) } - fn return_field_from_args( - &self, - args: ReturnFieldArgs, - ) -> datafusion_common::Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> datafusion_common::Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn invoke_with_args( &self, diff --git a/crates/spark/src/function/math/hypot.rs b/crates/spark/src/function/math/hypot.rs index 3a22a2fd..480e766b 100644 --- a/crates/spark/src/function/math/hypot.rs +++ b/crates/spark/src/function/math/hypot.rs @@ -4,16 +4,12 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, AsArray, Float64Array}; use arrow::compute::kernels::arity::binary; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Float64Type}; use datafusion_common::Result; use datafusion_common::utils::take_function_args; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -87,17 +83,7 @@ impl ScalarUDFImpl for SparkHypot { Ok(DataType::Float64) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/modulus.rs b/crates/spark/src/function/math/modulus.rs index 1fd1ae26..6e01773f 100644 --- a/crates/spark/src/function/math/modulus.rs +++ b/crates/spark/src/function/math/modulus.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, BooleanArray, Scalar, new_null_array}; use arrow::compute::kernels::cast::{CastOptions, cast, cast_with_options}; use arrow::compute::kernels::numeric::add_wrapping; @@ -14,13 +14,9 @@ use arrow::compute::kernels::{ numeric::{neg, rem}, zip::zip, }; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DECIMAL128_MAX_PRECISION, DataType}; use arrow::error::ArrowError; use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err, exec_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, binary::decimal_coercion, @@ -242,17 +238,7 @@ impl ScalarUDFImpl for SparkMod { Ok(arg_types[0].clone()) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_mod(&args.args, args.config_options.execution.enable_ansi_mode) @@ -319,17 +305,7 @@ impl ScalarUDFImpl for SparkPmod { } } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_pmod( diff --git a/crates/spark/src/function/math/negative.rs b/crates/spark/src/function/math/negative.rs index 88ee3998..1d55faa6 100644 --- a/crates/spark/src/function/math/negative.rs +++ b/crates/spark/src/function/math/negative.rs @@ -2,17 +2,13 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::types::*; use arrow::array::*; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit}; use bigdecimal::num_traits::WrappingNeg; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -95,17 +91,7 @@ impl ScalarUDFImpl for SparkNegative { Ok(arg_types[0].clone()) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/pow.rs b/crates/spark/src/function/math/pow.rs index 6432c76d..7a467d26 100644 --- a/crates/spark/src/function/math/pow.rs +++ b/crates/spark/src/function/math/pow.rs @@ -14,13 +14,9 @@ use arrow::array::{Array, ArrayRef, Float64Array}; use arrow::compute::binary; use arrow::datatypes::DataType; -use crate::function::nullability::SparkNullability; -use arrow::datatypes::FieldRef; +use crate::function::nullability::{SparkNullability, spark_nullability}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, }; @@ -78,17 +74,7 @@ impl ScalarUDFImpl for SparkPow { self.inner.return_type(arg_types) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { // Only Float64 × Float64 needs the Spark override. diff --git a/crates/spark/src/function/math/rint.rs b/crates/spark/src/function/math/rint.rs index 513a8617..2174790d 100644 --- a/crates/spark/src/function/math/rint.rs +++ b/crates/spark/src/function/math/rint.rs @@ -4,18 +4,14 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, AsArray}; use arrow::compute::cast; use arrow::datatypes::DataType::{ Float32, Float64, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, }; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Float32Type, Float64Type}; use datafusion_common::{Result, assert_eq_or_internal_err, exec_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -79,17 +75,7 @@ impl ScalarUDFImpl for SparkRint { Ok(Float64) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/unhex.rs b/crates/spark/src/function/math/unhex.rs index ec1e8298..e23c356e 100644 --- a/crates/spark/src/function/math/unhex.rs +++ b/crates/spark/src/function/math/unhex.rs @@ -2,10 +2,9 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, BinaryBuilder}; use arrow::datatypes::DataType; -use arrow::datatypes::FieldRef; use datafusion_common::cast::{ as_large_string_array, as_string_array, as_string_view_array, }; @@ -14,9 +13,6 @@ use datafusion_common::utils::{offset_span_len, take_function_args}; use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, }; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -86,17 +82,7 @@ impl ScalarUDFImpl for SparkUnhex { Ok(DataType::Binary) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/math/width_bucket.rs b/crates/spark/src/function/math/width_bucket.rs index 265623c6..f2391d5c 100644 --- a/crates/spark/src/function/math/width_bucket.rs +++ b/crates/spark/src/function/math/width_bucket.rs @@ -27,15 +27,11 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Int64Array, Int64Builder}; -use arrow::datatypes::FieldRef; use arrow::datatypes::TimeUnit::Microsecond; use datafusion_expr::Coercion; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; use datafusion_expr::Volatility::Immutable; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; /// Spark's `WidthBucket` is null-intolerant and always nullable, because invalid bounds or /// bucket counts return NULL. @@ -164,17 +160,7 @@ impl ScalarUDFImpl for SparkWidthBucket { Ok(Int64) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/nullability.rs b/crates/spark/src/function/nullability.rs index da12a0b5..b6d9198f 100644 --- a/crates/spark/src/function/nullability.rs +++ b/crates/spark/src/function/nullability.rs @@ -16,19 +16,24 @@ //! These are independent: a null-intolerant expression can still be nullable for non-NULL //! inputs. //! -//! A function here declares both in a [`SparkNullability`] and uses it from `ScalarUDFImpl`: +//! A function here declares both in a [`SparkNullability`] and uses it from `ScalarUDFImpl`, +//! where [`spark_nullability!`] implements `simplify`: //! //! ```ignore //! const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); //! -//! fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { -//! Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) -//! } +//! impl ScalarUDFImpl for SparkAscii { +//! fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { +//! Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) +//! } //! -//! fn simplify(&self, args: Vec, info: &SimplifyContext) -> Result { -//! NULLABILITY.simplify(self, args, info) +//! spark_nullability!(NULLABILITY); //! } //! ``` +//! +//! A function that computes its type in `return_type` uses +//! `spark_nullability!(NULLABILITY, return_type)`, which also implements +//! `return_field_from_args`. use std::sync::Arc; @@ -147,6 +152,37 @@ impl SparkNullability { } } +/// Implements `ScalarUDFImpl` methods from a function's [`SparkNullability`], inside its +/// `impl ScalarUDFImpl` block: +/// +/// - `spark_nullability!(NULLABILITY)` implements `simplify` with +/// [`SparkNullability::simplify`]. +/// - `spark_nullability!(NULLABILITY, return_type)` also implements `return_field_from_args` +/// with [`SparkNullability::return_field_from_type`], for a function that computes its type +/// in `return_type`. +macro_rules! spark_nullability { + ($nullability:expr) => { + fn simplify( + &self, + args: Vec, + info: &datafusion_expr::simplify::SimplifyContext, + ) -> datafusion_common::Result { + $nullability.simplify(self, args, info) + } + }; + ($nullability:expr, return_type) => { + $crate::function::nullability::spark_nullability!($nullability); + + fn return_field_from_args( + &self, + args: datafusion_expr::ReturnFieldArgs, + ) -> datafusion_common::Result { + $nullability.return_field_from_type(self, &args) + } + }; +} +pub(crate) use spark_nullability; + fn is_null_literal(expr: &Expr) -> bool { matches!(expr, Expr::Literal(value, _) if value.is_null()) } diff --git a/crates/spark/src/function/string/ascii.rs b/crates/spark/src/function/string/ascii.rs index 33a0a78c..eb8670a2 100644 --- a/crates/spark/src/function/string/ascii.rs +++ b/crates/spark/src/function/string/ascii.rs @@ -5,8 +5,6 @@ use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::types::{NativeType, logical_string}; use datafusion_common::{Result, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -15,7 +13,7 @@ use datafusion_functions::string::ascii::ascii; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `Ascii` is a null-intolerant `UnaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -89,13 +87,7 @@ impl ScalarUDFImpl for SparkAscii { Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/string/base64.rs b/crates/spark/src/function/string/base64.rs index 707a3c5d..55cc832c 100644 --- a/crates/spark/src/function/string/base64.rs +++ b/crates/spark/src/function/string/base64.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, GenericBinaryBuilder, OffsetSizeTrait}; use arrow::datatypes::DataType; use datafusion_common::arrow::datatypes::{Field, FieldRef}; @@ -198,13 +198,7 @@ impl ScalarUDFImpl for SparkUnBase64 { Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_unbase64, vec![])(&args.args) diff --git a/crates/spark/src/function/string/char.rs b/crates/spark/src/function/string/char.rs index ea59c1f2..6f8bf66a 100644 --- a/crates/spark/src/function/string/char.rs +++ b/crates/spark/src/function/string/char.rs @@ -11,14 +11,12 @@ use datafusion_macros::user_doc; use std::sync::Arc; use datafusion_common::{Result, ScalarValue, cast::as_int64_array, exec_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `Chr` is a null-intolerant `UnaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -91,13 +89,7 @@ impl ScalarUDFImpl for CharFunc { Ok(NULLABILITY.return_field(self.name(), Utf8, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); } /// Returns the ASCII character having the binary equivalent to the input expression. diff --git a/crates/spark/src/function/string/contains.rs b/crates/spark/src/function/string/contains.rs index f7fae26a..266d367d 100644 --- a/crates/spark/src/function/string/contains.rs +++ b/crates/spark/src/function/string/contains.rs @@ -4,17 +4,13 @@ use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{Array, ArrayRef, BooleanArray, Scalar}; use arrow::compute::kernels::cast::cast; use arrow::compute::kernels::comparison::contains as arrow_contains; use arrow::datatypes::DataType; -use arrow::datatypes::FieldRef; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -164,17 +160,7 @@ impl ScalarUDFImpl for SparkContains { Ok(DataType::Boolean) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/string/is_valid_utf8.rs b/crates/spark/src/function/string/is_valid_utf8.rs index 5379948c..6a4dd9c9 100644 --- a/crates/spark/src/function/string/is_valid_utf8.rs +++ b/crates/spark/src/function/string/is_valid_utf8.rs @@ -18,9 +18,7 @@ use datafusion_common::cast::{ use datafusion_common::utils::take_function_args; use datafusion_functions::utils::make_scalar_function; -use crate::function::nullability::SparkNullability; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use crate::function::nullability::{SparkNullability, spark_nullability}; use std::sync::Arc; /// Spark's `IsValidUTF8` is null-intolerant and always nullable. @@ -96,13 +94,7 @@ impl ScalarUDFImpl for SparkIsValidUtf8 { Ok(NULLABILITY.return_field(self.name(), DataType::Boolean, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/string/length.rs b/crates/spark/src/function/string/length.rs index cd1b49f4..e19248d6 100644 --- a/crates/spark/src/function/string/length.rs +++ b/crates/spark/src/function/string/length.rs @@ -7,8 +7,6 @@ use arrow::array::{ }; use arrow::datatypes::{DataType, FieldRef, Int32Type}; use datafusion_common::exec_err; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -17,7 +15,7 @@ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use std::sync::Arc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `Length` is a null-intolerant `UnaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -120,13 +118,7 @@ impl ScalarUDFImpl for SparkLengthFunc { Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> datafusion_common::Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); } fn spark_length(args: &[ArrayRef]) -> datafusion_common::Result { diff --git a/crates/spark/src/function/string/like.rs b/crates/spark/src/function/string/like.rs index f296613b..1e582af2 100644 --- a/crates/spark/src/function/string/like.rs +++ b/crates/spark/src/function/string/like.rs @@ -2,14 +2,12 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::ArrayRef; use arrow::compute::like; use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::ColumnarValue; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -89,13 +87,7 @@ impl ScalarUDFImpl for SparkLike { Ok(NULLABILITY.return_field(self.name(), DataType::Boolean, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_like, vec![])(&args.args) diff --git a/crates/spark/src/function/string/make_valid_utf8.rs b/crates/spark/src/function/string/make_valid_utf8.rs index a605f28e..60ec52f8 100644 --- a/crates/spark/src/function/string/make_valid_utf8.rs +++ b/crates/spark/src/function/string/make_valid_utf8.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, LargeStringArray, StringArray}; use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::{ @@ -10,8 +10,6 @@ use datafusion_common::cast::{ }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -102,13 +100,7 @@ impl ScalarUDFImpl for SparkMakeValidUtf8 { Ok(NULLABILITY.return_field(self.name(), return_type, args.arg_fields)) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); fn documentation(&self) -> Option<&Documentation> { self.doc() diff --git a/crates/spark/src/function/string/soundex.rs b/crates/spark/src/function/string/soundex.rs index 50c15d0f..d62631f6 100644 --- a/crates/spark/src/function/string/soundex.rs +++ b/crates/spark/src/function/string/soundex.rs @@ -2,16 +2,12 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait}; use arrow::datatypes::DataType; -use arrow::datatypes::FieldRef; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ColumnarValue, Documentation, Signature, Volatility}; use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl}; use datafusion_functions::utils::make_scalar_function; @@ -80,17 +76,7 @@ impl ScalarUDFImpl for SparkSoundex { } } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_soundex_inner, vec![])(&args.args) diff --git a/crates/spark/src/function/string/space.rs b/crates/spark/src/function/string/space.rs index dc026f46..e89dab24 100644 --- a/crates/spark/src/function/string/space.rs +++ b/crates/spark/src/function/string/space.rs @@ -2,18 +2,14 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, ArrayRef, DictionaryArray, Int32Array, StringArray, StringBuilder, as_dictionary_array, }; -use arrow::datatypes::FieldRef; use arrow::datatypes::{DataType, Int32Type}; use datafusion_common::cast::as_int32_array; use datafusion_common::{Result, ScalarValue, exec_err}; -use datafusion_expr::Expr; -use datafusion_expr::ReturnFieldArgs; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -94,17 +90,7 @@ impl ScalarUDFImpl for SparkSpace { Ok(return_type) } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - NULLABILITY.return_field_from_type(self, &args) - } - - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY, return_type); fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { spark_space(&args.args) diff --git a/crates/spark/src/function/string/substring.rs b/crates/spark/src/function/string/substring.rs index 4e6dc519..3f0d542b 100644 --- a/crates/spark/src/function/string/substring.rs +++ b/crates/spark/src/function/string/substring.rs @@ -14,8 +14,6 @@ use datafusion_common::types::{ NativeType, logical_int32, logical_int64, logical_string, }; use datafusion_common::{Result, exec_err}; -use datafusion_expr::Expr; -use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{Coercion, Documentation, ReturnFieldArgs, TypeSignatureClass}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, @@ -25,7 +23,7 @@ use datafusion_functions::unicode::substr::{enable_ascii_fast_path, get_true_sta use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; -use crate::function::nullability::SparkNullability; +use crate::function::nullability::{SparkNullability, spark_nullability}; /// Spark's `Substring` is a null-intolerant `TernaryExpression`. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -155,13 +153,7 @@ impl ScalarUDFImpl for SparkSubstring { )) } - fn simplify( - &self, - args: Vec, - info: &SimplifyContext, - ) -> Result { - NULLABILITY.simplify(self, args, info) - } + spark_nullability!(NULLABILITY); } fn spark_substring(args: &[ArrayRef]) -> Result { From 3802410dae6d1c9461caa7106511e49a21811f2f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 07:36:35 -0700 Subject: [PATCH 4/5] test(spark): cover array_contains nullability and test the like UDF The LIKE operator plans as DataFusion's built-in, so the like record now calls the like function only. Part of #383. --- .../src/function/array/array_contains.rs | 23 ++++++++++++++++++- .../test_files/predicate/like.slt | 8 +++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 502c6c6b..ed2d6c4d 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -184,9 +184,30 @@ fn mask_with_list_nulls( mod tests { use super::*; use arrow::array::{FixedSizeListArray, Float64Array}; - use arrow::datatypes::Field; use datafusion_common::ScalarValue; + fn return_field_nullable(array_nullable: bool, element_nullable: bool) -> bool { + let element = Arc::new(Field::new("item", DataType::Int64, element_nullable)); + let arg_fields = [ + Arc::new(Field::new("a", DataType::List(element), array_nullable)), + Arc::new(Field::new("v", DataType::Int64, false)), + ]; + SparkArrayContains::new() + .return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &[None, None], + }) + .unwrap() + .is_nullable() + } + + #[test] + fn return_field_is_nullable_when_the_array_can_hold_nulls() { + assert!(!return_field_nullable(false, false)); + assert!(return_field_nullable(true, false)); + assert!(return_field_nullable(false, true)); + } + #[test] fn zero_width_fixed_size_list_keeps_its_row() { // One valid row of a zero-width list, without a null buffer to infer the length from. diff --git a/crates/sqllogictest/test_files/predicate/like.slt b/crates/sqllogictest/test_files/predicate/like.slt index 297bf65e..a73c9781 100644 --- a/crates/sqllogictest/test_files/predicate/like.slt +++ b/crates/sqllogictest/test_files/predicate/like.slt @@ -71,9 +71,9 @@ SELECT like('Spark'::string, NULL::string); NULL # like is null-intolerant, as in Spark: a NULL argument makes every row NULL -query BB -SELECT column1 LIKE NULL::string, like(column1, NULL::string) +query B +SELECT like(column1, NULL::string) FROM VALUES ('Spark SQL'::string), (NULL::string); ---- -NULL NULL -NULL NULL +NULL +NULL From e6dae4a3ec554b0b697709bb5966f74ff2207adb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 07:41:13 -0700 Subject: [PATCH 5/5] refactor(spark): simplify SparkNullability after review - Make SparkNullability an enum of the two nullable rules, since every function that uses it is null-intolerant, and drop the unused flag. - Take &dyn ScalarUDFImpl, so the helpers are compiled once. - Remove the NULL-folding prologue from add_months, date_diff and unix_date: their rewrites already fold a NULL argument. - Share list_element_field from functions_nested_utils and use it in array_contains. - Move unix_seconds, unix_millis and unix_micros onto SparkNullability. Part of #383. --- .../src/function/array/array_contains.rs | 9 +-- .../spark/src/function/array/array_insert.rs | 11 +--- .../spark/src/function/datetime/add_months.rs | 4 -- .../spark/src/function/datetime/date_diff.rs | 4 -- .../src/function/datetime/make_dt_interval.rs | 46 ++----------- crates/spark/src/function/datetime/unix.rs | 14 ++-- .../src/function/functions_nested_utils.rs | 11 ++++ .../spark/src/function/map/map_from_arrays.rs | 2 - crates/spark/src/function/nullability.rs | 65 ++++++------------- 9 files changed, 47 insertions(+), 119 deletions(-) diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index ed2d6c4d..02c7c1dd 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::float_normalize::normalize_floats; +use crate::function::functions_nested_utils::list_element_field; use crate::function::nullability::{SparkNullability, spark_nullability}; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, @@ -66,12 +67,8 @@ impl ScalarUDFImpl for SparkArrayContains { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let contains_null = match args.arg_fields[0].data_type() { - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => field.is_nullable(), - _ => true, - }; + let contains_null = list_element_field(args.arg_fields[0].data_type()) + .is_none_or(|field| field.is_nullable()); let nullable = NULLABILITY.nullable(args.arg_fields) || contains_null; Ok(Arc::new(Field::new( self.name(), diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs index 28002d03..5862106e 100644 --- a/crates/spark/src/function/array/array_insert.rs +++ b/crates/spark/src/function/array/array_insert.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::function::functions_nested_utils::list_element_field; use std::sync::Arc; use arrow::array::{ @@ -103,16 +104,6 @@ impl SparkArrayInsert { } } -/// The element field of a list type. -fn list_element_field(data_type: &DataType) -> Option<&FieldRef> { - match data_type { - DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { - Some(f) - } - _ => None, - } -} - /// Spark's error when the first argument is not an array, or the item does /// not share a type with the array's elements. fn diff_types_err(array: &DataType, item: &DataType) -> DataFusionError { diff --git a/crates/spark/src/function/datetime/add_months.rs b/crates/spark/src/function/datetime/add_months.rs index b1888f52..8653e687 100644 --- a/crates/spark/src/function/datetime/add_months.rs +++ b/crates/spark/src/function/datetime/add_months.rs @@ -96,10 +96,6 @@ impl ScalarUDFImpl for SparkAddMonths { args: Vec, info: &SimplifyContext, ) -> Result { - let args = match NULLABILITY.simplify(self, args, info)? { - ExprSimplifyResult::Original(args) => args, - simplified => return Ok(simplified), - }; let [date_arg, months_arg] = take_function_args("add_months", args)?; let interval = months_arg .cast_to(&DataType::Interval(IntervalUnit::YearMonth), info.schema())?; diff --git a/crates/spark/src/function/datetime/date_diff.rs b/crates/spark/src/function/datetime/date_diff.rs index acd1299d..e41b3d92 100644 --- a/crates/spark/src/function/datetime/date_diff.rs +++ b/crates/spark/src/function/datetime/date_diff.rs @@ -124,10 +124,6 @@ impl ScalarUDFImpl for SparkDateDiff { args: Vec, info: &SimplifyContext, ) -> Result { - let args = match NULLABILITY.simplify(self, args, info)? { - ExprSimplifyResult::Original(args) => args, - simplified => return Ok(simplified), - }; let [end, start] = take_function_args(self.name(), args)?; let end = end.cast_to(&DataType::Date32, info.schema())?; let start = start.cast_to(&DataType::Date32, info.schema())?; diff --git a/crates/spark/src/function/datetime/make_dt_interval.rs b/crates/spark/src/function/datetime/make_dt_interval.rs index b740af21..450c7f37 100644 --- a/crates/spark/src/function/datetime/make_dt_interval.rs +++ b/crates/spark/src/function/datetime/make_dt_interval.rs @@ -342,55 +342,21 @@ mod tests { } #[test] - fn return_field_respects_nullability() -> Result<()> { - let udf = SparkMakeDtInterval::new(); - - // All nullable inputs -> nullable output + fn return_field_is_always_nullable() -> Result<()> { + // An overflow returns NULL (#20), so the result is nullable even for + // non-nullable inputs let arg_fields = vec![ - Arc::new(Field::new("days", DataType::Int32, true)), - Arc::new(Field::new("hours", DataType::Int32, true)), - Arc::new(Field::new("mins", DataType::Int32, true)), - Arc::new(Field::new("secs", DataType::Float64, true)), - ]; - - let out = udf.return_field_from_args(ReturnFieldArgs { - arg_fields: &arg_fields, - scalar_arguments: &[None, None, None, None], - })?; - assert!(out.is_nullable()); - assert_eq!(out.data_type(), &Duration(Microsecond)); - - // Non-nullable inputs -> nullable output, because an overflow returns NULL (#20) - let non_nullable_arg_fields = vec![ Arc::new(Field::new("days", DataType::Int32, false)), Arc::new(Field::new("hours", DataType::Int32, false)), Arc::new(Field::new("mins", DataType::Int32, false)), Arc::new(Field::new("secs", DataType::Float64, false)), ]; - - let out = udf.return_field_from_args(ReturnFieldArgs { - arg_fields: &non_nullable_arg_fields, + let out = SparkMakeDtInterval::new().return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, scalar_arguments: &[None, None, None, None], })?; assert!(out.is_nullable()); - - // Non-finite secs scalar is nullable too - let scalar_values = - [None, None, None, Some(ScalarValue::Float64(Some(f64::NAN)))]; - let scalar_refs = scalar_values.iter().map(|v| v.as_ref()).collect::>(); - let out = udf.return_field_from_args(ReturnFieldArgs { - arg_fields: &non_nullable_arg_fields, - scalar_arguments: &scalar_refs, - })?; - assert!(out.is_nullable()); - - // A zero-arg call (defaults) is nullable too, as the rule ignores the arguments - let out = udf.return_field_from_args(ReturnFieldArgs { - arg_fields: &[], - scalar_arguments: &[], - })?; - assert!(out.is_nullable()); - + assert_eq!(out.data_type(), &Duration(Microsecond)); Ok(()) } diff --git a/crates/spark/src/function/datetime/unix.rs b/crates/spark/src/function/datetime/unix.rs index e5b8c1e7..c0700ed3 100644 --- a/crates/spark/src/function/datetime/unix.rs +++ b/crates/spark/src/function/datetime/unix.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, LazyLock}; use crate::function::nullability::SparkNullability; -use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, TimeUnit}; +use arrow::datatypes::{DataType, FieldRef, Int64Type, TimeUnit}; use datafusion_common::cast::as_int64_array; use datafusion_common::types::logical_date; use datafusion_common::utils::take_function_args; @@ -18,7 +18,8 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; -/// Spark's `UnixDate` is a null-intolerant `UnaryExpression`. +/// Spark's `UnixDate`, `UnixSeconds`, `UnixMillis` and `UnixMicros` are null-intolerant +/// `UnaryExpression`s. const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); /// Spark-compatible `unix_date` expression. @@ -94,10 +95,6 @@ impl ScalarUDFImpl for SparkUnixDate { args: Vec, info: &SimplifyContext, ) -> Result { - let args = match NULLABILITY.simplify(self, args, info)? { - ExprSimplifyResult::Original(args) => args, - simplified => return Ok(simplified), - }; let [date] = take_function_args(self.name(), args)?; Ok(ExprSimplifyResult::Simplified( date.cast_to(&DataType::Date32, info.schema())? @@ -180,8 +177,7 @@ impl ScalarUDFImpl for SparkUnixTimestamp { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields[0].is_nullable(); - Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable))) + Ok(NULLABILITY.return_field(self.name(), DataType::Int64, args.arg_fields)) } fn documentation(&self) -> Option<&Documentation> { @@ -271,7 +267,7 @@ NULL .downscale_divisor(&ts.get_type(info.schema())?) .is_some() { - return Ok(ExprSimplifyResult::Original(vec![ts])); + return NULLABILITY.simplify(self, vec![ts], info); } Ok(ExprSimplifyResult::Simplified( ts.cast_to( diff --git a/crates/spark/src/function/functions_nested_utils.rs b/crates/spark/src/function/functions_nested_utils.rs index a6188de2..e8ef96ba 100644 --- a/crates/spark/src/function/functions_nested_utils.rs +++ b/crates/spark/src/function/functions_nested_utils.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use arrow::array::{Array, ArrayRef}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::ColumnarValue; @@ -38,3 +39,13 @@ where } } } + +/// The element field of a list type. +pub(crate) fn list_element_field(data_type: &DataType) -> Option<&FieldRef> { + match data_type { + DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { + Some(f) + } + _ => None, + } +} diff --git a/crates/spark/src/function/map/map_from_arrays.rs b/crates/spark/src/function/map/map_from_arrays.rs index 2a86831c..98def462 100644 --- a/crates/spark/src/function/map/map_from_arrays.rs +++ b/crates/spark/src/function/map/map_from_arrays.rs @@ -65,8 +65,6 @@ impl ScalarUDFImpl for MapFromArrays { get_element_type(keys_field.data_type())?, get_element_type(values_field.data_type())?, ); - // Spark marks map_from_arrays as null intolerant, so the output is - // nullable if either input is nullable. Ok(NULLABILITY.return_field(self.name(), map_type, args.arg_fields)) } diff --git a/crates/spark/src/function/nullability.rs b/crates/spark/src/function/nullability.rs index b6d9198f..07fd7ad9 100644 --- a/crates/spark/src/function/nullability.rs +++ b/crates/spark/src/function/nullability.rs @@ -16,8 +16,8 @@ //! These are independent: a null-intolerant expression can still be nullable for non-NULL //! inputs. //! -//! A function here declares both in a [`SparkNullability`] and uses it from `ScalarUDFImpl`, -//! where [`spark_nullability!`] implements `simplify`: +//! Every function that uses [`SparkNullability`] is null-intolerant, so it declares only the +//! `nullable` rule, and [`spark_nullability!`] implements `simplify` from it: //! //! ```ignore //! const NULLABILITY: SparkNullability = SparkNullability::null_intolerant(); @@ -42,50 +42,36 @@ use datafusion_common::{Result, ScalarValue}; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{Expr, ExprSchemable, ReturnFieldArgs, ScalarUDFImpl, lit}; -/// When a function's result can be NULL: Spark's `Expression.nullable`. +/// The nullability of a null-intolerant function: when its result can be NULL, as Spark's +/// `Expression.nullable` declares it. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) enum SparkNullable { - /// When any argument can be NULL: `children.exists(_.nullable)`, the default of +pub(crate) enum SparkNullability { + /// Nullable when any argument is: `children.exists(_.nullable)`, the default of /// `UnaryExpression`, `BinaryExpression` and `TernaryExpression`. Children, - /// Always, because the function can return NULL for non-NULL arguments. + /// Always nullable, because the function can return NULL for non-NULL arguments. Always, } -/// A function's nullability: Spark's `nullable` and `nullIntolerant`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) struct SparkNullability { - nullable: SparkNullable, - null_intolerant: bool, -} - impl SparkNullability { /// A null-intolerant function whose result is nullable when an argument is. This is a /// `UnaryExpression`, `BinaryExpression` or `TernaryExpression` with /// `nullIntolerant = true` that does not override `nullable`. pub(crate) const fn null_intolerant() -> Self { - Self::new(SparkNullable::Children, true) + Self::Children } /// A null-intolerant function that can return NULL for non-NULL arguments, so its result /// is always nullable. Spark overrides `nullable` to `true` for these. pub(crate) const fn null_intolerant_always_nullable() -> Self { - Self::new(SparkNullable::Always, true) - } - - /// Declares `nullable` and `nullIntolerant` separately. - pub(crate) const fn new(nullable: SparkNullable, null_intolerant: bool) -> Self { - Self { - nullable, - null_intolerant, - } + Self::Always } /// Whether the result can be NULL, given the argument fields. pub(crate) fn nullable(&self, arg_fields: &[FieldRef]) -> bool { - match self.nullable { - SparkNullable::Children => arg_fields.iter().any(|f| f.is_nullable()), - SparkNullable::Always => true, + match self { + Self::Children => arg_fields.iter().any(|f| f.is_nullable()), + Self::Always => true, } } @@ -101,9 +87,9 @@ impl SparkNullability { /// The return field of a function that computes its type in `ScalarUDFImpl::return_type`, /// for `ScalarUDFImpl::return_field_from_args`. - pub(crate) fn return_field_from_type( + pub(crate) fn return_field_from_type( &self, - func: &F, + func: &dyn ScalarUDFImpl, args: &ReturnFieldArgs, ) -> Result { let arg_types = args @@ -115,18 +101,18 @@ impl SparkNullability { Ok(self.return_field(func.name(), data_type, args.arg_fields)) } - /// For `ScalarUDFImpl::simplify`: replaces a call to a null-intolerant function that has - /// a NULL literal argument with a NULL literal of its return type, as Spark's - /// `NullPropagation` does. Otherwise returns the arguments unchanged. + /// For `ScalarUDFImpl::simplify`: replaces a call that has a NULL literal argument with a + /// NULL literal of its return type, as Spark's `NullPropagation` does for a + /// null-intolerant expression. Otherwise returns the arguments unchanged. /// /// This is only an optimization: `invoke_with_args` must return NULL for such a call too. - pub(crate) fn simplify( + pub(crate) fn simplify( &self, - func: &F, + func: &dyn ScalarUDFImpl, args: Vec, info: &SimplifyContext, ) -> Result { - if !self.null_intolerant || !args.iter().any(is_null_literal) { + if !args.iter().any(is_null_literal) { return Ok(ExprSimplifyResult::Original(args)); } let arg_fields = args @@ -144,7 +130,7 @@ impl SparkNullability { arg_fields: &arg_fields, scalar_arguments: &scalar_arguments, })?; - match ScalarValue::try_from(return_field.data_type()) { + match ScalarValue::try_new_null(return_field.data_type()) { Ok(null) => Ok(ExprSimplifyResult::Simplified(lit(null))), // A type that has no NULL `ScalarValue` is left to `invoke_with_args` Err(_) => Ok(ExprSimplifyResult::Original(args)), @@ -268,13 +254,4 @@ mod tests { assert_eq!(original, args); } } - - #[test] - fn test_simplify_not_null_intolerant() { - let args = vec![lit(ScalarValue::Utf8(None))]; - let result = SparkNullability::new(SparkNullable::Children, false) - .simplify(&SparkAscii::new(), args.clone(), &schema()) - .unwrap(); - assert!(matches!(result, ExprSimplifyResult::Original(a) if a == args)); - } }