Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 43 additions & 2 deletions crates/spark/src/function/array/array_contains.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,25 @@
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use crate::function::float_normalize::normalize_floats;
use crate::function::functions_nested_utils::list_element_field;
use crate::function::nullability::{SparkNullability, spark_nullability};
use arrow::array::{
Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait,
};
use arrow::buffer::{BooleanBuffer, NullBuffer};
use arrow::datatypes::DataType;
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::{Result, exec_err};
use datafusion_expr::ReturnFieldArgs;
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
};
use datafusion_functions_nested::array_has::array_has_udf;
use std::sync::Arc;

/// Spark's `ArrayContains` is a null-intolerant `BinaryExpression`. It is also nullable
/// when the array can hold NULL elements.
const NULLABILITY: SparkNullability = SparkNullability::null_intolerant();

/// Spark-compatible `array_contains` function.
///
/// Calls DataFusion's `array_has` and then applies Spark's null semantics:
Expand Down Expand Up @@ -59,6 +66,19 @@ impl ScalarUDFImpl for SparkArrayContains {
Ok(DataType::Boolean)
}

fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
let contains_null = list_element_field(args.arg_fields[0].data_type())
.is_none_or(|field| field.is_nullable());
let nullable = NULLABILITY.nullable(args.arg_fields) || contains_null;
Ok(Arc::new(Field::new(
self.name(),
DataType::Boolean,
nullable,
)))
}

spark_nullability!(NULLABILITY);

fn invoke_with_args(&self, mut args: ScalarFunctionArgs) -> Result<ColumnarValue> {
for arg in &mut args.args {
*arg = normalize_floats(arg)?;
Expand Down Expand Up @@ -161,9 +181,30 @@ fn mask_with_list_nulls(
mod tests {
use super::*;
use arrow::array::{FixedSizeListArray, Float64Array};
use arrow::datatypes::Field;
use datafusion_common::ScalarValue;

fn return_field_nullable(array_nullable: bool, element_nullable: bool) -> bool {
let element = Arc::new(Field::new("item", DataType::Int64, element_nullable));
let arg_fields = [
Arc::new(Field::new("a", DataType::List(element), array_nullable)),
Arc::new(Field::new("v", DataType::Int64, false)),
];
SparkArrayContains::new()
.return_field_from_args(ReturnFieldArgs {
arg_fields: &arg_fields,
scalar_arguments: &[None, None],
})
.unwrap()
.is_nullable()
}

#[test]
fn return_field_is_nullable_when_the_array_can_hold_nulls() {
assert!(!return_field_nullable(false, false));
assert!(return_field_nullable(true, false));
assert!(return_field_nullable(false, true));
}

#[test]
fn zero_width_fixed_size_list_keeps_its_row() {
// One valid row of a zero-width list, without a null buffer to infer the length from.
Expand Down
9 changes: 9 additions & 0 deletions crates/spark/src/function/array/array_extrema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
//! Spark-compatible `array_max` and `array_min`.

use crate::function::float_normalize::normalize_array;
use crate::function::nullability::{SparkNullability, spark_nullability};
use crate::function::type_coercion::{is_orderable, spark_sql_type};
use arrow::array::{
Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray, OffsetSizeTrait,
Expand All @@ -26,6 +27,10 @@ use datafusion_macros::user_doc;
use std::cmp::Ordering;
use std::sync::Arc;

/// Spark's `ArrayMax` and `ArrayMin` are null-intolerant and always nullable, because an
/// empty array returns NULL.
const NULLABILITY: SparkNullability = SparkNullability::null_intolerant_always_nullable();

/// Spark-compatible `array_max` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_max>
Expand Down Expand Up @@ -82,6 +87,8 @@ impl ScalarUDFImpl for SparkArrayMax {
element_type(self.name(), arg_types)
}

spark_nullability!(NULLABILITY, return_type);

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
coerce_array(self.name(), arg_types)
}
Expand Down Expand Up @@ -151,6 +158,8 @@ impl ScalarUDFImpl for SparkArrayMin {
element_type(self.name(), arg_types)
}

spark_nullability!(NULLABILITY, return_type);

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
coerce_array(self.name(), arg_types)
}
Expand Down
11 changes: 1 addition & 10 deletions crates/spark/src/function/array/array_insert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use crate::function::functions_nested_utils::list_element_field;
use std::sync::Arc;

use arrow::array::{
Expand Down Expand Up @@ -103,16 +104,6 @@ impl SparkArrayInsert {
}
}

/// The element field of a list type.
fn list_element_field(data_type: &DataType) -> Option<&FieldRef> {
match data_type {
DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => {
Some(f)
}
_ => None,
}
}

/// Spark's error when the first argument is not an array, or the item does
/// not share a type with the array's elements.
fn diff_types_err(array: &DataType, item: &DataType) -> DataFusionError {
Expand Down
6 changes: 6 additions & 0 deletions crates/spark/src/function/array/array_position.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use crate::function::float_normalize::normalize_array;
use crate::function::nullability::{SparkNullability, spark_nullability};
use crate::function::type_coercion::{
is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type,
};
Expand All @@ -21,6 +22,9 @@ use datafusion_expr::{
use datafusion_macros::user_doc;
use std::sync::Arc;

/// Spark's `ArrayPosition` is a null-intolerant `BinaryExpression`.
const NULLABILITY: SparkNullability = SparkNullability::null_intolerant();

/// Spark-compatible `array_position` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_position>
Expand Down Expand Up @@ -85,6 +89,8 @@ impl ScalarUDFImpl for SparkArrayPosition {
Ok(DataType::Int64)
}

spark_nullability!(NULLABILITY, return_type);

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
let [array, element] = take_function_args(self.name(), arg_types)?;
coerce_array_and_element(self.name(), array, element)
Expand Down
6 changes: 6 additions & 0 deletions crates/spark/src/function/array/array_remove.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use crate::function::float_normalize::normalize_array;
use crate::function::nullability::{SparkNullability, spark_nullability};
use crate::function::type_coercion::{
is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type,
};
Expand All @@ -24,6 +25,9 @@ use datafusion_macros::user_doc;
use num_traits::Float;
use std::sync::Arc;

/// Spark's `ArrayRemove` is a null-intolerant `BinaryExpression`.
const NULLABILITY: SparkNullability = SparkNullability::null_intolerant();

/// Spark-compatible `array_remove` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
Expand Down Expand Up @@ -85,6 +89,8 @@ impl ScalarUDFImpl for SparkArrayRemove {
Ok(array.clone())
}

spark_nullability!(NULLABILITY, return_type);

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
let [array, element] = take_function_args(self.name(), arg_types)?;
coerce_array_and_element(self.name(), array, element)
Expand Down
11 changes: 8 additions & 3 deletions crates/spark/src/function/array/slice.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ use datafusion_functions_nested::extract::array_slice_udf;
use datafusion_macros::user_doc;
use std::sync::Arc;

use crate::function::nullability::{SparkNullability, spark_nullability};

/// Spark's `Slice` is a null-intolerant `TernaryExpression`.
const NULLABILITY: SparkNullability = SparkNullability::null_intolerant();

/// Spark-compatible `slice` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#slice>
Expand Down Expand Up @@ -94,18 +99,18 @@ impl ScalarUDFImpl for SparkSlice {
}

fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
let nullable = args.arg_fields.iter().any(|f| f.is_nullable());

let data_type = match args.arg_fields[0].data_type() {
DataType::Null => {
DataType::List(Arc::new(Field::new_list_field(DataType::Null, true)))
}
dt => dt.clone(),
};

Ok(Arc::new(Field::new("slice", data_type, nullable)))
Ok(NULLABILITY.return_field("slice", data_type, args.arg_fields))
}

spark_nullability!(NULLABILITY);

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
Expand Down
14 changes: 7 additions & 7 deletions crates/spark/src/function/bitwise/bit_count.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

use std::sync::Arc;

use crate::function::nullability::{SparkNullability, spark_nullability};
use arrow::array::{ArrayRef, AsArray, Int32Array};
use arrow::datatypes::{
DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type,
Expand All @@ -18,6 +19,9 @@ use datafusion_expr::{
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;

/// Spark's `BitwiseCount` is a null-intolerant `UnaryExpression`.
const NULLABILITY: SparkNullability = SparkNullability::null_intolerant();

/// Spark-compatible `bit_count` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#bit_count>
Expand Down Expand Up @@ -91,15 +95,11 @@ impl ScalarUDFImpl for SparkBitCount {
&self,
args: datafusion_expr::ReturnFieldArgs,
) -> Result<FieldRef> {
use arrow::datatypes::Field;
// bit_count returns Int32 with the same nullability as the input
Ok(Arc::new(Field::new(
args.arg_fields[0].name(),
DataType::Int32,
args.arg_fields[0].is_nullable(),
)))
Ok(NULLABILITY.return_field(self.name(), DataType::Int32, args.arg_fields))
}

spark_nullability!(NULLABILITY);

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
Expand Down
14 changes: 10 additions & 4 deletions crates/spark/src/function/bitwise/bit_get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use arrow::array::{
downcast_integer_array,
};
use arrow::compute::try_binary;
use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef, Int8Type, Int32Type};
use arrow::datatypes::{ArrowNativeType, DataType, FieldRef, Int8Type, Int32Type};
use datafusion_common::types::{NativeType, logical_int32};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
Expand All @@ -21,6 +21,11 @@ use datafusion_expr::{
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;

use crate::function::nullability::{SparkNullability, spark_nullability};

/// Spark's `BitwiseGet` is a null-intolerant `BinaryExpression`.
const NULLABILITY: SparkNullability = SparkNullability::null_intolerant();

/// Spark-compatible `bit_get` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#bit_get>
Expand Down Expand Up @@ -97,11 +102,11 @@ impl ScalarUDFImpl for SparkBitGet {
}

fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
// Spark derives nullability for BinaryExpression from its children
let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
Ok(Arc::new(Field::new(self.name(), DataType::Int8, nullable)))
Ok(NULLABILITY.return_field(self.name(), DataType::Int8, args.arg_fields))
}

spark_nullability!(NULLABILITY);

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
Expand Down Expand Up @@ -142,6 +147,7 @@ fn spark_bit_get(args: &[ArrayRef]) -> Result<ArrayRef> {
#[cfg(test)]
mod tests {
use super::*;
use arrow::datatypes::Field;

#[test]
fn test_bit_get_nullability_non_nullable_inputs() {
Expand Down
14 changes: 10 additions & 4 deletions crates/spark/src/function/bitwise/bit_shift.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@

use std::sync::Arc;

use crate::function::nullability::{SparkNullability, spark_nullability};
use arrow::array::{ArrayRef, ArrowPrimitiveType, AsArray, Int32Array, PrimitiveArray};
use arrow::compute;
use arrow::datatypes::{
ArrowNativeType, DataType, Field, FieldRef, Int32Type, Int64Type, UInt32Type,
UInt64Type,
ArrowNativeType, DataType, FieldRef, Int32Type, Int64Type, UInt32Type, UInt64Type,
};
use datafusion_common::types::{
NativeType, logical_int8, logical_int16, logical_int32, logical_int64, logical_uint8,
Expand Down Expand Up @@ -180,6 +180,10 @@ enum BitShiftType {
RightUnsigned,
}

/// Spark's `ShiftLeft`, `ShiftRight` and `ShiftRightUnsigned` are null-intolerant
/// `BinaryExpression`s.
const NULLABILITY: SparkNullability = SparkNullability::null_intolerant();

/// Spark-compatible `shiftleft`, `shiftright` and `shiftrightunsigned` expressions.
///
/// One struct implements all three functions, so each function's documentation is
Expand Down Expand Up @@ -353,11 +357,12 @@ impl ScalarUDFImpl for SparkBitShift {
}

fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
let data_type = args.arg_fields[0].data_type().clone();
Ok(Arc::new(Field::new(self.name(), data_type, nullable)))
Ok(NULLABILITY.return_field(self.name(), data_type, args.arg_fields))
}

spark_nullability!(NULLABILITY);

fn documentation(&self) -> Option<&Documentation> {
match self.bit_shift_type {
BitShiftType::Left => ShiftLeftDoc.doc(),
Expand All @@ -377,6 +382,7 @@ impl ScalarUDFImpl for SparkBitShift {
#[cfg(test)]
mod tests {
use super::*;
use arrow::datatypes::Field;

#[test]
fn test_bit_shift_nullability() -> Result<()> {
Expand Down
Loading
Loading