diff --git a/crates/spark/src/function/math/log1p.rs b/crates/spark/src/function/math/log1p.rs new file mode 100644 index 00000000..cde08648 --- /dev/null +++ b/crates/spark/src/function/math/log1p.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; +use arrow::array::{ArrayRef, AsArray, Float64Array}; +use arrow::datatypes::{DataType, Float64Type}; +use datafusion_common::config::ConfigOptions; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, internal_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, + Signature, Volatility, +}; +use datafusion_macros::user_doc; +use std::sync::Arc; + +/// Spark-compatible `log1p` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns `ln(1 + expr)` as a DOUBLE, computed accurately when `expr` is \ + close to 0. Values less than or equal to -1 return NULL. NULL input returns NULL.", + syntax_example = "log1p(expr)", + sql_example = r#"```sql +> SELECT log1p(0); +0 +> SELECT log1p(-1); +NULL +```"#, + argument( + name = "expr", + description = "Numeric expression evaluated as DOUBLE. Values <= -1 and NULL return NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLog1p { + input_types: SparkInputTypes, +} + +impl Default for SparkLog1p { + fn default() -> Self { + Self::new() + } +} + +impl SparkLog1p { + pub fn new() -> Self { + Self { + input_types: SparkInputTypes::new( + vec![SparkInputType::Double], + Volatility::Immutable, + ), + } + } +} + +impl ScalarUDFImpl for SparkLog1p { + fn name(&self) -> &str { + "log1p" + } + + fn signature(&self) -> &Signature { + self.input_types.signature() + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + self.input_types.coerce(self.name(), arg_types) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + let input_types = self.input_types.with_config(config)?; + Some(ScalarUDF::from(Self { input_types })) + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let [arg] = take_function_args(self.name(), args.args)?; + + match arg { + ColumnarValue::Scalar(ScalarValue::Float64(value)) => { + Ok(ColumnarValue::Scalar(ScalarValue::Float64( + value.and_then(log1p_or_null), + ))) + } + ColumnarValue::Array(array) => { + let result: Float64Array = array + .as_primitive::() + .iter() + .map(|value| value.and_then(log1p_or_null)) + .collect(); + + Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef)) + } + other => internal_err!( + "Unexpected argument type for log1p after coercion: {}", + other.data_type() + ), + } + } +} + +fn log1p_or_null(value: f64) -> Option { + if value <= -1.0 { + None + } else { + Some(value.ln_1p()) + } +} diff --git a/crates/spark/src/function/math/mod.rs b/crates/spark/src/function/math/mod.rs index 129d40d4..9aa4b730 100644 --- a/crates/spark/src/function/math/mod.rs +++ b/crates/spark/src/function/math/mod.rs @@ -16,6 +16,7 @@ pub mod floor; pub mod greatest_least; pub mod hex; pub mod hypot; +pub mod log1p; pub mod modulus; pub mod negative; pub mod positive; @@ -54,6 +55,7 @@ make_udf_function!(negative::SparkNegative, negative); make_udf_function!(bin::SparkBin, bin); make_udf_function!(bround::SparkBRound, bround); make_udf_function!(divide::SparkDivide, spark_divide); +make_udf_function!(log1p::SparkLog1p, log1p); pub mod expr_fn { use datafusion_functions::export_functions; @@ -117,6 +119,11 @@ pub mod expr_fn { "Rounds the value of expr to scale decimal places using HALF_EVEN rounding mode.", arg1 arg2 )); + export_functions!(( + log1p, + "Returns the natural logarithm of 1 plus the given value, computed accurately for values near zero. Returns NULL when the input is less than or equal to -1.", + arg1 + )); } pub fn functions() -> Vec> { @@ -145,5 +152,6 @@ pub fn functions() -> Vec> { bin(), bround(), spark_divide(), + log1p(), ] } diff --git a/crates/sqllogictest/test_files/math/log1p.slt b/crates/sqllogictest/test_files/math/log1p.slt index 1bde6500..a55e2821 100644 --- a/crates/sqllogictest/test_files/math/log1p.slt +++ b/crates/sqllogictest/test_files/math/log1p.slt @@ -2,13 +2,104 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -# This file was originally created by a porting script from: -# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function -# This file is part of the implementation of the datafusion-spark function library. -# For more information, please see: -# https://github.com/apache/datafusion/issues/15914 - -## Original Query: SELECT log1p(0); -## PySpark 3.5.5 Result: {'LOG1P(0)': 0.0, 'typeof(LOG1P(0))': 'double', 'typeof(0)': 'int'} -#query -#SELECT log1p(0::int); +# Basic values and a negative input within the domain +query R +SELECT log1p(0.5::DOUBLE); +---- +0.4054651081081644 + +query R +SELECT log1p(1.0::DOUBLE); +---- +0.6931471805599453 + +query R +SELECT log1p(-0.5::DOUBLE); +---- +-0.6931471805599453 + +# A value immediately greater than the boundary -1 +query R +SELECT log1p(-0.9999999999999999::DOUBLE); +---- +-36.7368005696771 + +# The boundary and values outside the domain return NULL +query R +SELECT log1p(-1.0::DOUBLE); +---- +NULL + +query R +SELECT log1p(-2.0::DOUBLE); +---- +NULL + +query R +SELECT log1p(CAST('-Infinity' AS DOUBLE)); +---- +NULL + +# NULL input returns NULL +query R +SELECT log1p(CAST(NULL AS DOUBLE)); +---- +NULL + +# Evaluate multiple values in a column +query R +SELECT log1p(a) +FROM (VALUES + (0.0::DOUBLE), + (0.5::DOUBLE), + (-0.5::DOUBLE), + (-1.0::DOUBLE) +) AS t(a); +---- +0 +0.4054651081081644 +-0.6931471805599453 +NULL + +# Small inputs retain precision near zero +query B +SELECT log1p(1e-300::DOUBLE) = 1e-300::DOUBLE; +---- +true + +query B +SELECT log1p(1e-20::DOUBLE) = 1e-20::DOUBLE; +---- +true + +# NaN and positive infinity are preserved +query R +SELECT log1p(CAST('NaN' AS DOUBLE)); +---- +NaN + +query R +SELECT log1p(CAST('Infinity' AS DOUBLE)); +---- +Infinity + +# Integer, float, decimal, and string inputs +query R +SELECT log1p(CAST(1 AS INT)); +---- +0.6931471805599453 + +query R +SELECT log1p(CAST(0.5 AS FLOAT)); +---- +0.4054651081081644 + +query R +SELECT log1p(CAST(0.5 AS DECIMAL(10, 2))); +---- +0.4054651081081644 + +query R +SELECT log1p('0.5'); +---- +0.4054651081081644 diff --git a/docs/functions/README.md b/docs/functions/README.md index 2532c7f7..8e50ed5e 100644 --- a/docs/functions/README.md +++ b/docs/functions/README.md @@ -17,7 +17,7 @@ function differs from the DataFusion built-in of the same name, if there is one. ## Scalar Functions -- [Math Functions](scalar/math.md) (27) +- [Math Functions](scalar/math.md) (28) - [Conditional Functions](scalar/conditional.md) (2) - [String Functions](scalar/string.md) (17) - [Binary String Functions](scalar/binary_string.md) (5) @@ -102,6 +102,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [len](scalar/string.md#len) - [length](scalar/string.md#length) - [like](scalar/string.md#like) +- [log1p](scalar/math.md#log1p) - [luhn_check](scalar/string.md#luhn_check) - [make_date](scalar/time_and_date.md#make_date) - [make_dt_interval](scalar/time_and_date.md#make_dt_interval) diff --git a/docs/functions/scalar/math.md b/docs/functions/scalar/math.md index ce76c2f9..f43b12a6 100644 --- a/docs/functions/scalar/math.md +++ b/docs/functions/scalar/math.md @@ -32,6 +32,7 @@ Part of the [datafusion-spark function reference](../README.md). - [hex](#hex) - [hypot](#hypot) - [least](#least) +- [log1p](#log1p) - [negative](#negative) - [positive](#positive) - [pow](#pow) @@ -437,6 +438,27 @@ least(expr1, expr2[, ..., expr_n]) 1.0 ``` +## `log1p` + +Returns `ln(1 + expr)` as a DOUBLE, computed accurately when `expr` is close to 0. Values less than or equal to -1 return NULL. NULL input returns NULL. + +```sql +log1p(expr) +``` + +### Arguments + +- **expr**: Numeric expression evaluated as DOUBLE. Values <= -1 and NULL return NULL. + +### Example + +```sql +> SELECT log1p(0); +0 +> SELECT log1p(-1); +NULL +``` + ## `negative` Returns the negation of `expr` (the same as unary minus), with the same type as the input. DataFusion has no built-in `negative`. Accepts integer, floating-point, decimal and interval values. Negating the minimum value of a signed integer type (for example `-2147483648` for INT) depends on ANSI mode: with `datafusion.execution.enable_ansi_mode` off it wraps around and returns the same value, and with it on the function fails with an overflow error. Returns NULL if the input is NULL and NaN if it is NaN.