diff --git a/crates/spark/src/function/string/find_in_set.rs b/crates/spark/src/function/string/find_in_set.rs new file mode 100644 index 00000000..9f94c10c --- /dev/null +++ b/crates/spark/src/function/string/find_in_set.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::Result; +use datafusion_common::config::ConfigOptions; +use datafusion_common::utils::take_function_args; +use datafusion_expr::{ + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, +}; +use datafusion_functions::unicode::find_in_set as core_find_in_set; +use datafusion_macros::user_doc; + +use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; + +/// Spark-compatible `find_in_set` expression. +/// +#[user_doc( + doc_section(label = "String Functions"), + description = "Returns the 1-based index of the first matching element as an INT. \ + Uses the UTF8_BINARY collation: the search is case-sensitive and elements are split on commas without trimming. \ + Empty elements count, including the single empty element in an empty list. \ + Returns 0 when the string contains a comma or has no match and NULL if either argument is NULL.", + syntax_example = "find_in_set(str, strlist)", + sql_example = r#"```sql +> SELECT find_in_set('b', 'a,b,c'); +2 +> SELECT find_in_set('', 'a,,b'); +2 +```"#, + argument(name = "str", description = "String expression to match."), + argument(name = "strlist", description = "Comma-separated list of strings.") +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkFindInSet { + input_types: SparkInputTypes, +} + +impl Default for SparkFindInSet { + fn default() -> Self { + Self::new() + } +} + +impl SparkFindInSet { + pub fn new() -> Self { + let input_types = SparkInputTypes::new( + vec![SparkInputType::String, SparkInputType::String], + Volatility::Immutable, + ); + Self { input_types } + } +} + +impl ScalarUDFImpl for SparkFindInSet { + fn name(&self) -> &str { + "find_in_set" + } + + fn signature(&self) -> &Signature { + self.input_types.signature() + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let types = self.input_types.coerce(self.name(), arg_types)?; + // The core kernel expects both string arrays to use the same layout. + let target = if types.contains(&DataType::LargeUtf8) { + DataType::LargeUtf8 + } else if types.contains(&DataType::Utf8View) { + DataType::Utf8View + } else { + DataType::Utf8 + }; + Ok(vec![target; types.len()]) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + let input_types = self.input_types.with_config(config)?; + Some(ScalarUDF::from(Self { input_types })) + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int32) + } + + fn return_field_from_args(&self, args: ReturnFieldArgs<'_>) -> Result { + let nullable = args.arg_fields.iter().any(|field| field.is_nullable()); + Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, mut args: ScalarFunctionArgs) -> Result { + let core = core_find_in_set(); + let [needle, list] = take_function_args(self.name(), &args.args)?; + let return_type = core.return_type(&[needle.data_type(), list.data_type()])?; + if return_type == DataType::Int32 { + return core.invoke_with_args(args); + } + + // DataFusion returns Int64 for LargeUtf8. Spark always returns INT. + args.return_field = Arc::new( + args.return_field + .as_ref() + .clone() + .with_data_type(return_type), + ); + core.invoke_with_args(args)?.cast_to(&DataType::Int32, None) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn return_field_matches_spark_nullability() -> Result<()> { + let function = SparkFindInSet::new(); + for (needle_nullable, list_nullable) in + [(false, false), (true, false), (false, true)] + { + let fields = [ + Arc::new(Field::new("str", DataType::LargeUtf8, needle_nullable)), + Arc::new(Field::new("strlist", DataType::LargeUtf8, list_nullable)), + ]; + let result = function.return_field_from_args(ReturnFieldArgs { + arg_fields: &fields, + scalar_arguments: &[None, None], + })?; + assert_eq!(result.data_type(), &DataType::Int32); + assert_eq!(result.is_nullable(), needle_nullable || list_nullable); + } + Ok(()) + } +} diff --git a/crates/spark/src/function/string/mod.rs b/crates/spark/src/function/string/mod.rs index b9729acd..810beb80 100644 --- a/crates/spark/src/function/string/mod.rs +++ b/crates/spark/src/function/string/mod.rs @@ -13,6 +13,7 @@ pub mod concat_ws; pub mod contains; pub mod elt; pub mod encode; +pub mod find_in_set; pub mod format_string; pub mod ilike; pub mod is_valid_utf8; @@ -40,6 +41,7 @@ make_udf_function!(ilike::SparkILike, ilike); make_udf_function!(length::SparkLengthFunc, length); make_udf_function!(elt::SparkElt, elt); make_udf_function!(encode::SparkEncode, encode); +make_udf_function!(find_in_set::SparkFindInSet, find_in_set); make_udf_function!(like::SparkLike, like); make_udf_function!(luhn_check::SparkLuhnCheck, luhn_check); make_udf_function!(format_string::FormatStringFunc, format_string); @@ -96,6 +98,11 @@ pub mod expr_fn { "Encodes a string or binary value into binary using the specified character encoding.", string_or_binary charset )); + export_functions!(( + find_in_set, + "Returns the 1-based position of the first match in a comma-separated list, or 0.", + str strlist + )); export_functions!(( ilike, "Returns true if str matches pattern (case insensitive).", @@ -170,6 +177,7 @@ pub fn functions() -> Vec> { contains(), elt(), encode(), + find_in_set(), ilike(), length(), like(), diff --git a/crates/sqllogictest/test_files/string/find_in_set.slt b/crates/sqllogictest/test_files/string/find_in_set.slt index bae2a4bc..a943d06c 100644 --- a/crates/sqllogictest/test_files/string/find_in_set.slt +++ b/crates/sqllogictest/test_files/string/find_in_set.slt @@ -220,3 +220,80 @@ SELECT find_in_set(arrow_cast(a, 'LargeUtf8'), arrow_cast('abc,b,ab,c,def', 'Lar 3 0 NULL + +# Spark implicitly casts both arguments to strings. +query II +SELECT find_in_set(2::INT, '1,2,3'), find_in_set('2', 2::INT); +---- +2 1 + +query I +SELECT find_in_set(a, 'false,true') +FROM (VALUES (true), (false), (NULL::BOOLEAN)) AS t(a); +---- +2 +1 +NULL + +query I +SELECT find_in_set(X'62', 'a,b'); +---- +2 + +query I +SELECT find_in_set(DATE '2020-01-01', 'x,2020-01-01'); +---- +2 + +# xfail spark: timestamp-to-string casting differs (https://github.com/apache/datafusion-spark/issues/88). +query I +SELECT find_in_set(TIMESTAMP '2020-01-01 00:00:00', 'x,2020-01-01 00:00:00'); +---- +0 + +# xfail spark: double-to-string casting differs (https://github.com/apache/datafusion-spark/issues/88). +query I +SELECT find_in_set(1e20::DOUBLE, '1.0E20,100000000000000000000'); +---- +0 + +# xfail spark: binary-to-string casting differs (https://github.com/apache/datafusion-spark/issues/88). +statement error Encountered non UTF-8 data +SELECT find_in_set(X'FF', 'a,b'); + +query III +SELECT find_in_set(NULL, 'a'), find_in_set('a', NULL), find_in_set(NULL, NULL); +---- +NULL NULL NULL + +query I +SELECT find_in_set(arrow_cast(a, 'LargeUtf8'), arrow_cast(b, 'Utf8View')) +FROM (VALUES ('b', 'a,b'), ('界', 'a,界'), (NULL::STRING, 'a,b')) AS t(a, b); +---- +2 +2 +NULL + +query I +SELECT find_in_set(arrow_cast(a, 'Dictionary(Int8, Utf8)'), b) +FROM (VALUES ('b', 'a,b'), ('a', 'a,a'), (NULL::STRING, 'a,b')) AS t(a, b); +---- +2 +1 +NULL + +query T +SELECT arrow_typeof(find_in_set(arrow_cast('b', 'LargeUtf8'), 'a,b')); +---- +Int32 + +query T rowsort +SELECT arrow_typeof(find_in_set(arrow_cast(a, 'LargeUtf8'), b)) +FROM (VALUES ('b', 'a,b'), (NULL::STRING, 'a,b')) AS t(a, b); +---- +Int32 +Int32 + +# xfail spark: collations are not supported yet (https://github.com/apache/datafusion-spark/issues/299). +statement error Unsupported ast node in sqltorel: Collate +SELECT find_in_set('B' COLLATE UTF8_LCASE, 'a,b'); diff --git a/docs/functions/README.md b/docs/functions/README.md index e8e68a9b..2a435f94 100644 --- a/docs/functions/README.md +++ b/docs/functions/README.md @@ -19,9 +19,9 @@ function differs from the DataFusion built-in of the same name, if there is one. - [Math Functions](scalar/math.md) (27) - [Conditional Functions](scalar/conditional.md) (2) -- [String Functions](scalar/string.md) (17) +- [String Functions](scalar/string.md) (18) - [Binary String Functions](scalar/binary_string.md) (5) -- [Time and Date Functions](scalar/time_and_date.md) (28) +- [Time and Date Functions](scalar/time_and_date.md) (29) - [Array Functions](scalar/array.md) (10) - [Hashing Functions](scalar/hashing.md) (4) - [Other Functions](scalar/other.md) (6) @@ -77,11 +77,13 @@ function differs from the DataFusion built-in of the same name, if there is one. - [datediff](scalar/time_and_date.md#datediff) - [datepart](scalar/time_and_date.md#datepart) - [dayname](scalar/time_and_date.md#dayname) +- [dayofweek](scalar/time_and_date.md#dayofweek) - [elt](scalar/string.md#elt) - [encode](scalar/binary_string.md#encode) - [equal_null](scalar/other.md#equal_null) - [expm1](scalar/math.md#expm1) - [factorial](scalar/math.md#factorial) +- [find_in_set](scalar/string.md#find_in_set) - [floor](scalar/math.md#floor) - [format_string](scalar/string.md#format_string) - [from_utc_timestamp](scalar/time_and_date.md#from_utc_timestamp) diff --git a/docs/functions/scalar/string.md b/docs/functions/scalar/string.md index 09fffe61..ebd58e22 100644 --- a/docs/functions/scalar/string.md +++ b/docs/functions/scalar/string.md @@ -22,6 +22,7 @@ Part of the [datafusion-spark function reference](../README.md). - [concat_ws](#concat_ws) - [contains](#contains) - [elt](#elt) +- [find_in_set](#find_in_set) - [format_string](#format_string) - [ilike](#ilike) - [is_valid_utf8](#is_valid_utf8) @@ -188,6 +189,28 @@ scala NULL ``` +## `find_in_set` + +Returns the 1-based index of the first matching element as an INT. Uses the UTF8_BINARY collation: the search is case-sensitive and elements are split on commas without trimming. Empty elements count, including the single empty element in an empty list. Returns 0 when the string contains a comma or has no match and NULL if either argument is NULL. + +```sql +find_in_set(str, strlist) +``` + +### Arguments + +- **str**: String expression to match. +- **strlist**: Comma-separated list of strings. + +### Example + +```sql +> SELECT find_in_set('b', 'a,b,c'); +2 +> SELECT find_in_set('', 'a,,b'); +2 +``` + ## `format_string` Returns a string formatted from a printf-style format string, following Java's `java.util.Formatter` as Spark does. DataFusion has no built-in equivalent. Supports the `%s`, `%b`, `%c`, `%d`, `%o`, `%x`, `%e`, `%f`, `%g`, `%a` and `%t` (date/time) conversions and their uppercase forms, the `-`, `+`, ` `, `0`, `#`, `,` and `(` flags, width and precision, and positional (`%2$s`) and relative (`% SELECT dayofweek('2009-07-30'::DATE); +5 +> SELECT dayofweek('2009-07-30'); +5 +> SELECT dayofweek(arrow_cast('2024-01-07 23:59:59', 'Timestamp(Microsecond, None)')); +1 +> SELECT dayofweek(NULL::DATE); +NULL +``` + ## `from_utc_timestamp` Treats `ts` as a UTC wall-clock time and returns the wall-clock time in time zone `tz`. DataFusion has no built-in equivalent. A string `ts` is cast to a timestamp, so a string with an offset is first converted to UTC. The result has the same type as `ts`, including any time zone it carries. Returns NULL if either argument is NULL. `tz` accepts Spark's spellings: region IDs, offsets such as `+01:00`, `GMT+1`, `UTC+1:2` and `UT-010203`, Java short IDs such as `PST`, and legacy `SystemV/` IDs. An offset beyond 18 hours or an unknown zone is an error. Differences from Spark: `tz` is only parsed for rows where both arguments are non-NULL, so an invalid constant `tz` can return NULL where Spark raises an error; conversion is limited to Chrono's calendar range, which is narrower than the range Spark implements but covers Spark's documented range; and region rules from `chrono-tz` may differ after 2099, because it does not extrapolate later daylight saving time transitions.