Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 117 additions & 0 deletions crates/spark/src/function/math/log1p.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use crate::function::implicit_cast::{SparkInputType, SparkInputTypes};
use arrow::array::{ArrayRef, AsArray, Float64Array};
use arrow::datatypes::{DataType, Float64Type};
use datafusion_common::config::ConfigOptions;
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
Signature, Volatility,
};
use datafusion_macros::user_doc;
use std::sync::Arc;

/// Spark-compatible `log1p` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#log1p>
#[user_doc(
doc_section(label = "Math Functions"),
description = "Returns `ln(1 + expr)` as a DOUBLE, computed accurately when `expr` is \
close to 0. Values less than or equal to -1 return NULL. NULL input returns NULL.",
syntax_example = "log1p(expr)",
sql_example = r#"```sql
> SELECT log1p(0);
0
> SELECT log1p(-1);
NULL
```"#,
argument(
name = "expr",
description = "Numeric expression evaluated as DOUBLE. Values <= -1 and NULL return NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkLog1p {
input_types: SparkInputTypes,
}

impl Default for SparkLog1p {
fn default() -> Self {
Self::new()
}
}

impl SparkLog1p {
pub fn new() -> Self {
Self {
input_types: SparkInputTypes::new(
vec![SparkInputType::Double],
Volatility::Immutable,
),
}
}
}

impl ScalarUDFImpl for SparkLog1p {
fn name(&self) -> &str {
"log1p"
}

fn signature(&self) -> &Signature {
self.input_types.signature()
}

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
self.input_types.coerce(self.name(), arg_types)
}

fn with_updated_config(&self, config: &ConfigOptions) -> Option<ScalarUDF> {
let input_types = self.input_types.with_config(config)?;
Some(ScalarUDF::from(Self { input_types }))
}

fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
Ok(DataType::Float64)
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let [arg] = take_function_args(self.name(), args.args)?;

match arg {
ColumnarValue::Scalar(ScalarValue::Float64(value)) => {
Ok(ColumnarValue::Scalar(ScalarValue::Float64(
value.and_then(log1p_or_null),
)))
}
ColumnarValue::Array(array) => {
let result: Float64Array = array
.as_primitive::<Float64Type>()
.iter()
.map(|value| value.and_then(log1p_or_null))
.collect();

Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef))
}
other => internal_err!(
"Unexpected argument type for log1p after coercion: {}",
other.data_type()
),
}
}
}

fn log1p_or_null(value: f64) -> Option<f64> {
if value <= -1.0 {
None
} else {
Some(value.ln_1p())
}
}
8 changes: 8 additions & 0 deletions crates/spark/src/function/math/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ pub mod floor;
pub mod greatest_least;
pub mod hex;
pub mod hypot;
pub mod log1p;
pub mod modulus;
pub mod negative;
pub mod positive;
Expand Down Expand Up @@ -54,6 +55,7 @@ make_udf_function!(negative::SparkNegative, negative);
make_udf_function!(bin::SparkBin, bin);
make_udf_function!(bround::SparkBRound, bround);
make_udf_function!(divide::SparkDivide, spark_divide);
make_udf_function!(log1p::SparkLog1p, log1p);

pub mod expr_fn {
use datafusion_functions::export_functions;
Expand Down Expand Up @@ -117,6 +119,11 @@ pub mod expr_fn {
"Rounds the value of expr to scale decimal places using HALF_EVEN rounding mode.",
arg1 arg2
));
export_functions!((
log1p,
"Returns the natural logarithm of 1 plus the given value, computed accurately for values near zero. Returns NULL when the input is less than or equal to -1.",
arg1
));
}

pub fn functions() -> Vec<Arc<ScalarUDF>> {
Expand Down Expand Up @@ -145,5 +152,6 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
bin(),
bround(),
spark_divide(),
log1p(),
]
}
111 changes: 101 additions & 10 deletions crates/sqllogictest/test_files/math/log1p.slt
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,104 @@
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

# This file was originally created by a porting script from:
# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
# This file is part of the implementation of the datafusion-spark function library.
# For more information, please see:
# https://github.com/apache/datafusion/issues/15914

## Original Query: SELECT log1p(0);
## PySpark 3.5.5 Result: {'LOG1P(0)': 0.0, 'typeof(LOG1P(0))': 'double', 'typeof(0)': 'int'}
#query
#SELECT log1p(0::int);
# Basic values and a negative input within the domain
query R
SELECT log1p(0.5::DOUBLE);
----
0.4054651081081644

query R
SELECT log1p(1.0::DOUBLE);
----
0.6931471805599453

query R
SELECT log1p(-0.5::DOUBLE);
----
-0.6931471805599453

# A value immediately greater than the boundary -1
query R
SELECT log1p(-0.9999999999999999::DOUBLE);
----
-36.7368005696771

# The boundary and values outside the domain return NULL
query R
SELECT log1p(-1.0::DOUBLE);
----
NULL

query R
SELECT log1p(-2.0::DOUBLE);
----
NULL

query R
SELECT log1p(CAST('-Infinity' AS DOUBLE));
----
NULL

# NULL input returns NULL
query R
SELECT log1p(CAST(NULL AS DOUBLE));
----
NULL

# Evaluate multiple values in a column
query R
SELECT log1p(a)
FROM (VALUES
(0.0::DOUBLE),
(0.5::DOUBLE),
(-0.5::DOUBLE),
(-1.0::DOUBLE)
) AS t(a);
----
0
0.4054651081081644
-0.6931471805599453
NULL

# Small inputs retain precision near zero
query B
SELECT log1p(1e-300::DOUBLE) = 1e-300::DOUBLE;
----
true

query B
SELECT log1p(1e-20::DOUBLE) = 1e-20::DOUBLE;
----
true

# NaN and positive infinity are preserved
query R
SELECT log1p(CAST('NaN' AS DOUBLE));
----
NaN

query R
SELECT log1p(CAST('Infinity' AS DOUBLE));
----
Infinity

# Integer, float, decimal, and string inputs
query R
SELECT log1p(CAST(1 AS INT));
----
0.6931471805599453

query R
SELECT log1p(CAST(0.5 AS FLOAT));
----
0.4054651081081644

query R
SELECT log1p(CAST(0.5 AS DECIMAL(10, 2)));
----
0.4054651081081644

query R
SELECT log1p('0.5');
----
0.4054651081081644

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a few cases for the inputs Spark rejects, like log1p(true), log1p(DATE '2020-01-01') and log1p(X'01'), as query error? A malformed string like log1p('abc') would also be useful. It returns NULL with ANSI off and raises CAST_INVALID_INPUT with ANSI on, so it needs the ANSI wrapper described in test_files/README.md.

3 changes: 2 additions & 1 deletion docs/functions/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ function differs from the DataFusion built-in of the same name, if there is one.

## Scalar Functions

- [Math Functions](scalar/math.md) (27)
- [Math Functions](scalar/math.md) (28)
- [Conditional Functions](scalar/conditional.md) (2)
- [String Functions](scalar/string.md) (17)
- [Binary String Functions](scalar/binary_string.md) (5)
Expand Down Expand Up @@ -102,6 +102,7 @@ function differs from the DataFusion built-in of the same name, if there is one.
- [len](scalar/string.md#len)
- [length](scalar/string.md#length)
- [like](scalar/string.md#like)
- [log1p](scalar/math.md#log1p)
- [luhn_check](scalar/string.md#luhn_check)
- [make_date](scalar/time_and_date.md#make_date)
- [make_dt_interval](scalar/time_and_date.md#make_dt_interval)
Expand Down
22 changes: 22 additions & 0 deletions docs/functions/scalar/math.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ Part of the [datafusion-spark function reference](../README.md).
- [hex](#hex)
- [hypot](#hypot)
- [least](#least)
- [log1p](#log1p)
- [negative](#negative)
- [positive](#positive)
- [pow](#pow)
Expand Down Expand Up @@ -437,6 +438,27 @@ least(expr1, expr2[, ..., expr_n])
1.0
```

## `log1p`

Returns `ln(1 + expr)` as a DOUBLE, computed accurately when `expr` is close to 0. Values less than or equal to -1 return NULL. NULL input returns NULL.

```sql
log1p(expr)
```

### Arguments

- **expr**: Numeric expression evaluated as DOUBLE. Values <= -1 and NULL return NULL.

### Example

```sql
> SELECT log1p(0);
0
> SELECT log1p(-1);
NULL
```

## `negative`

Returns the negation of `expr` (the same as unary minus), with the same type as the input. DataFusion has no built-in `negative`. Accepts integer, floating-point, decimal and interval values. Negating the minimum value of a signed integer type (for example `-2147483648` for INT) depends on ANSI mode: with `datafusion.execution.enable_ansi_mode` off it wraps around and returns the same value, and with it on the function fails with an overflow error. Returns NULL if the input is NULL and NaN if it is NaN.
Expand Down