Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 30 additions & 7 deletions crates/spark/src/function/string/ascii.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,37 @@ use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::types::{NativeType, logical_string};
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature, TypeSignatureClass, Volatility,
Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
};
use datafusion_functions::string::ascii::ascii;
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;

/// Spark compatible version of the [ascii] function. Differs from the [default ascii function]
/// in that it is more permissive of input types, for example casting numeric input to string
/// before executing the function (default version doesn't allow numeric input).
/// Spark-compatible `ascii` expression.
///
/// [ascii]: https://spark.apache.org/docs/latest/api/sql/index.html#ascii
/// [default ascii function]: datafusion_functions::string::ascii::AsciiFunc
/// <https://spark.apache.org/docs/latest/api/sql/index.html#ascii>
#[user_doc(
doc_section(label = "String Functions"),
description = "Returns the numeric code point of the first character of `str`. \
Despite the name, this is the full Unicode code point, not just ASCII (`ascii('😀')` is 128512). \
An empty string returns 0, and NULL returns NULL. \
Unlike DataFusion's `ascii`, numeric input is accepted and cast to a string first, \
so `ascii(2)` is the code point of `'2'`.",
syntax_example = "ascii(str)",
sql_example = r#"```sql
> SELECT ascii('Spark');
83
> SELECT ascii('');
0
> SELECT ascii(2);
50
```"#,
argument(
name = "str",
description = "String expression. Numeric values are cast to a string. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkAscii {
signature: Signature,
Expand Down Expand Up @@ -68,6 +87,10 @@ impl ScalarUDFImpl for SparkAscii {
Ok(Arc::new(Field::new("ascii", DataType::Int32, nullable)))
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
make_scalar_function(ascii, vec![])(&args.args)
}
Expand Down
31 changes: 28 additions & 3 deletions crates/spark/src/function/string/char.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,37 @@ use arrow::array::GenericStringBuilder;
use arrow::datatypes::DataType::Int64;
use arrow::datatypes::DataType::Utf8;
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_macros::user_doc;
use std::sync::Arc;

use datafusion_common::{Result, ScalarValue, cast::as_int64_array, exec_err};
use datafusion_expr::{
ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature, Volatility,
};

/// Spark-compatible `char` expression
/// Spark-compatible `char` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#char>
#[user_doc(
doc_section(label = "String Functions"),
description = "Returns the character whose code point is `n` modulo 256, so `char(321)` is `A`. \
A negative `n` returns an empty string, and NULL returns NULL. \
DataFusion's `chr` instead uses `n` as a full Unicode code point and errors on negative or invalid values.",
syntax_example = "char(n)",
sql_example = r#"```sql
> SELECT char(65);
A
> SELECT char(321);
A
> SELECT char(-1);
-- empty string
```"#,
argument(
name = "n",
description = "Integer expression, cast to BIGINT. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct CharFunc {
signature: Signature,
Expand All @@ -37,6 +58,10 @@ impl CharFunc {
}

impl ScalarUDFImpl for CharFunc {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn name(&self) -> &str {
"char"
}
Expand Down
37 changes: 28 additions & 9 deletions crates/spark/src/function/string/concat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,39 @@ use datafusion_common::arrow::datatypes::FieldRef;
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::ReturnFieldArgs;
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
use datafusion_functions::string::concat::ConcatFunc;
use datafusion_macros::user_doc;
use std::sync::Arc;

use crate::function::null_utils::{
NullMaskResolution, apply_null_mask, compute_null_mask,
};

/// Spark-compatible `concat` expression
/// <https://spark.apache.org/docs/latest/api/sql/index.html#concat>
///
/// Concatenates multiple input strings into a single string.
/// Returns NULL if any input is NULL.
/// Spark-compatible `concat` expression.
///
/// Differences with DataFusion concat:
/// - Support 0 arguments
/// - Return NULL if any input is NULL
/// <https://spark.apache.org/docs/latest/api/sql/index.html#concat>
#[user_doc(
doc_section(label = "String Functions"),
description = "Concatenates the arguments into a single string, or binary if all arguments are binary. \
Returns NULL if any argument is NULL, whereas DataFusion's `concat` skips NULL arguments. \
Unlike DataFusion's `concat`, it also accepts zero arguments and then returns an empty string.",
syntax_example = "concat([expr1[, ..., exprN]])",
sql_example = r#"```sql
> SELECT concat('Spark', 'SQL');
SparkSQL
> SELECT concat('Spark', 'SQL', NULL);
NULL
> SELECT concat();
-- empty string
```"#,
argument(
name = "expr",
description = "String or binary expressions to concatenate. Any NULL makes the result NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkConcat {
signature: Signature,
Expand All @@ -45,6 +60,10 @@ impl SparkConcat {
}

impl ScalarUDFImpl for SparkConcat {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn name(&self) -> &str {
"concat"
}
Expand Down
49 changes: 38 additions & 11 deletions crates/spark/src/function/string/concat_ws.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,8 @@
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! Spark-compatible `concat_ws`: joins strings (and array elements) with a separator.
//!
//! Null scalar args and null array elements are skipped; a null separator yields a
//! null row. Non-string args are coerced to STRING; list args (`List`, `LargeList`,
//! `ListView`, `LargeListView`, `FixedSizeList`) expand their elements.
//!
//! Differences with DataFusion core `concat_ws`:
//! - Accepts list arguments and expands their elements
//! - Always returns Utf8 (Spark's `STRING` type)
//! - Coerces non-string scalars (numbers, booleans, dates, ...) to Utf8

use datafusion_macros::user_doc;
use std::fmt::Write as _;
use std::sync::Arc;

Expand All @@ -24,14 +16,45 @@ use arrow::datatypes::{DataType, Field};
use datafusion_common::arrow::datatypes::FieldRef;
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{
ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature, Volatility,
};

use crate::function::error_utils::{
invalid_arg_count_exec_err, unsupported_data_type_exec_err,
};

/// Spark-compatible `concat_ws` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#concat_ws>
#[user_doc(
doc_section(label = "String Functions"),
description = "Concatenates the arguments after the first, separated by `sep`. \
NULL arguments and NULL array elements are skipped, so if every argument is NULL the result is an empty string. \
A NULL separator returns NULL. With only a separator, the result is an empty string. \
Unlike DataFusion's `concat_ws`, array arguments are expanded into their elements, \
non-string arguments (numbers, booleans, dates, binary, array elements) are cast to strings, \
and the result is always `Utf8`.",
syntax_example = "concat_ws(sep[, expr1[, ..., exprN]])",
sql_example = r#"```sql
> SELECT concat_ws(' ', 'Spark', 'SQL');
Spark SQL
> SELECT concat_ws('/', 'foo', NULL, 'bar');
foo/bar
> SELECT concat_ws(',', 'x', array('a', 'b'), 'y');
x,a,b,y
> SELECT concat_ws(NULL, 'a', 'b');
NULL
```"#,
argument(
name = "sep",
description = "Separator, cast to a string. NULL returns NULL."
),
argument(
name = "expr",
description = "Values or arrays to join. Arrays contribute each element. NULL values and elements are skipped. Structs are not supported."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkConcatWs {
signature: Signature,
Expand All @@ -52,6 +75,10 @@ impl SparkConcatWs {
}

impl ScalarUDFImpl for SparkConcatWs {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn name(&self) -> &str {
"concat_ws"
}
Expand Down
36 changes: 35 additions & 1 deletion crates/spark/src/function/string/elt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use datafusion_macros::user_doc;
use std::sync::Arc;

use arrow::array::{
Expand All @@ -13,10 +14,39 @@ use arrow::datatypes::{DataType, Int64Type};
use datafusion_common::cast::as_string_array;
use datafusion_common::{DataFusionError, Result, exec_err, plan_datafusion_err};
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
use datafusion_functions::utils::make_scalar_function;

/// Spark-compatible `elt` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#elt>
#[user_doc(
doc_section(label = "String Functions"),
description = "Returns the `n`-th of the following arguments as a string (1-based). \
DataFusion has no built-in `elt`. \
A NULL `n` returns NULL, as does picking a NULL argument. \
If `n` is less than 1 or greater than the number of arguments, the result is NULL, \
or an error when `datafusion.execution.enable_ansi_mode` is true.",
syntax_example = "elt(n, expr1[, ..., exprN])",
sql_example = r#"```sql
> SELECT elt(1, 'scala', 'java');
scala
> SELECT elt(2, 'a', 1);
1
> SELECT elt(3, 'scala', 'java');
NULL
```"#,
argument(
name = "n",
description = "Integer index, 1-based, cast to BIGINT. NULL returns NULL."
),
argument(
name = "expr",
description = "At least one value to choose from. Each is cast to a string."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkElt {
signature: Signature,
Expand All @@ -37,6 +67,10 @@ impl SparkElt {
}

impl ScalarUDFImpl for SparkElt {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn name(&self) -> &str {
"elt"
}
Expand Down
43 changes: 40 additions & 3 deletions crates/spark/src/function/string/format_string.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use datafusion_macros::user_doc;
use std::fmt::Write;
use std::sync::Arc;

Expand All @@ -20,12 +21,44 @@ use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, plan_err,
};
use datafusion_expr::{
ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
TypeSignature, Volatility,
ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature, TypeSignature, Volatility,
};

/// Spark-compatible `format_string` expression
/// Spark-compatible `format_string` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#format_string>
#[user_doc(
doc_section(label = "String Functions"),
description = "Returns a string formatted from a printf-style format string, following Java's `java.util.Formatter` \
as Spark does. DataFusion has no built-in equivalent. \
Supports the `%s`, `%b`, `%c`, `%d`, `%o`, `%x`, `%e`, `%f`, `%g`, `%a` and `%t` (date/time) conversions \
and their uppercase forms, the `-`, `+`, ` `, `0`, `#`, `,` and `(` flags, width and precision, \
and positional (`%2$s`) and relative (`%<s`) argument references. \
A NULL format string returns NULL, while a NULL argument is formatted as `null`. \
It is an error if there are too few arguments or an argument does not fit its conversion. \
`%h` and `%H` (hash code) are not supported and return an error.",
syntax_example = "format_string(format[, arg1[, ..., argN]])",
alternative_syntax = "printf(format[, arg1[, ..., argN]])",
sql_example = r#"```sql
> SELECT format_string('Hello World %d %s', 100, 'days');
Hello World 100 days
> SELECT format_string('%2$s %1$d', 42, 'test');
test 42
> SELECT format_string('Precision: %.2f', 3.14159);
Precision: 3.14
> SELECT format_string('Value: %s', NULL);
Value: null
```"#,
argument(
name = "format",
description = "String format specification. NULL returns NULL."
),
argument(
name = "arg",
description = "Values to format. Each must suit the conversion that refers to it. A NULL value is formatted as `null`."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct FormatStringFunc {
signature: Signature,
Expand All @@ -48,6 +81,10 @@ impl FormatStringFunc {
}

impl ScalarUDFImpl for FormatStringFunc {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn name(&self) -> &str {
"format_string"
}
Expand Down
Loading
Loading