diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec981b4a..8916c710 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,6 +44,10 @@ details. `inputTypes` with `SparkInputTypes` (`function/implicit_cast.rs`), which supplies `signature()`, `coerce_types()` and `with_updated_config()` with Spark's errors and ANSI rules. See `sha2`. It only chooses the target types, so the cast that then runs is not yet Spark's. +- **Checked input types.** When the expression uses `ExpectsInputTypes` without implicit casts, + use `SparkInputTypes::check_only`. It shares the declared-type validation and Spark errors, + accepts untyped NULLs, and unwraps dictionaries. Widen accepted inputs separately when the + execution kernel needs a common representation. See `timestamp_micros`. - **Common type of several arguments.** When Spark finds one type for several arguments, as `array_position` does, call the rules in `function/type_coercion.rs` (for example `tightest_common_type()`) from `coerce_types()`. diff --git a/crates/spark/src/function/datetime/mod.rs b/crates/spark/src/function/datetime/mod.rs index aff5ac5e..6f140ddb 100644 --- a/crates/spark/src/function/datetime/mod.rs +++ b/crates/spark/src/function/datetime/mod.rs @@ -24,6 +24,7 @@ pub mod make_time; pub mod monthname; pub mod next_day; pub mod time_trunc; +pub mod timestamp_micros; pub(crate) mod timezone; pub mod to_utc_timestamp; pub mod trunc; @@ -60,6 +61,7 @@ make_udf_function!(make_time::SparkMakeTime, make_time); make_udf_function!(monthname::SparkMonthName, monthname); make_udf_function!(next_day::SparkNextDay, next_day); make_udf_function!(time_trunc::SparkTimeTrunc, time_trunc); +make_udf_function!(timestamp_micros::SparkTimestampMicros, timestamp_micros); make_udf_function!(to_utc_timestamp::SparkToUtcTimestamp, to_utc_timestamp); make_udf_function!(trunc::SparkTrunc, trunc); make_udf_function!(unix::SparkUnixDate, unix_date); @@ -186,6 +188,11 @@ pub mod expr_fn { "Interpret a given timestamp `ts` in UTC timezone and then convert it to timezone `tz`.", ts tz )); + export_functions!(( + timestamp_micros, + "Creates a timestamp from microseconds since the UTC epoch.", + microseconds + )); export_functions!(( to_utc_timestamp, "Interpret a given timestamp `ts` in timezone `tz` and then convert it to UTC timezone.", @@ -240,6 +247,7 @@ pub fn functions() -> Vec> { next_day(), second(), time_trunc(), + timestamp_micros(), to_utc_timestamp(), trunc(), unix_date(), diff --git a/crates/spark/src/function/datetime/timestamp_micros.rs b/crates/spark/src/function/datetime/timestamp_micros.rs new file mode 100644 index 00000000..b2c21f48 --- /dev/null +++ b/crates/spark/src/function/datetime/timestamp_micros.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit}; +use datafusion_common::config::ConfigOptions; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, internal_err, plan_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, +}; +use datafusion_macros::user_doc; + +use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; + +/// Spark-compatible `timestamp_micros` expression. +/// +/// +#[user_doc( + doc_section(label = "Time and Date Functions"), + description = "Returns the timestamp that is `microseconds` microseconds after \ + 1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike DataFusion's \ + `to_timestamp_micros`, only integer types whose full range fits in BIGINT are \ + accepted; floating-point, decimal and string inputs are rejected. The result has microsecond precision \ + and the session time zone (`datafusion.execution.time_zone`, default UTC). \ + The time zone affects display, not the stored microsecond count. Every BIGINT \ + value is valid, including its minimum and maximum. Returns NULL for NULL input.", + syntax_example = "timestamp_micros(microseconds)", + sql_example = r#"```sql +> SELECT timestamp_micros(0::BIGINT); +1970-01-01T00:00:00Z +> SELECT timestamp_micros(-1::BIGINT); +1969-12-31T23:59:59.999999Z +> SELECT timestamp_micros(NULL); +NULL +```"#, + argument( + name = "microseconds", + description = "TINYINT, SMALLINT, INT or BIGINT count of microseconds since the UTC epoch. Arrow UInt8, UInt16 and UInt32 inputs are also accepted. UInt64 is rejected because its full range cannot fit in BIGINT. NULL returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkTimestampMicros { + input_types: SparkInputTypes, + timezone: Arc, +} + +impl Default for SparkTimestampMicros { + fn default() -> Self { + Self::new() + } +} + +impl SparkTimestampMicros { + pub fn new() -> Self { + Self::new_with_config(&ConfigOptions::default()) + } + + pub fn new_with_config(config: &ConfigOptions) -> Self { + Self { + input_types: SparkInputTypes::check_only( + vec![SparkInputType::Integral], + Volatility::Stable, + ), + timezone: config + .execution + .time_zone + .as_ref() + .map_or("UTC", |tz| tz.as_str()) + .into(), + } + } +} + +impl ScalarUDFImpl for SparkTimestampMicros { + fn name(&self) -> &str { + "timestamp_micros" + } + + fn signature(&self) -> &Signature { + self.input_types.signature() + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let types = self.input_types.coerce(self.name(), arg_types)?; + if types[0] == DataType::UInt64 { + return plan_err!( + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of \ + `timestamp_micros` requires an integer type whose full range fits in \ + BIGINT, however it has the type \"UInt64\". SQLSTATE: 42K09" + ); + } + Ok(vec![DataType::Int64]) + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let [input] = take_function_args(self.name(), args.arg_fields)?; + Ok(Arc::new(Field::new( + self.name(), + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::clone(&self.timezone))), + input.is_nullable(), + ))) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + Some(ScalarUDF::from(Self::new_with_config(config))) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let [microseconds] = take_function_args(self.name(), args.args)?; + // Both types store an i64 count, so this does not scale the value or + // convert it through a calendar representation (which has a smaller range). + microseconds.cast_to( + &DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::clone(&self.timezone))), + None, + ) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn return_field_tracks_session_timezone_and_input_nullability() -> Result<()> { + for timezone in [None, Some("America/Los_Angeles")] { + let mut config = ConfigOptions::default(); + config.execution.time_zone = timezone.map(str::parse).transpose()?; + let function = SparkTimestampMicros::new() + .with_updated_config(&config) + .unwrap(); + for nullable in [false, true] { + let input = Arc::new(Field::new("micros", DataType::Int64, nullable)); + let field = function.inner().return_field_from_args(ReturnFieldArgs { + arg_fields: &[input], + scalar_arguments: &[None], + })?; + assert_eq!(field.is_nullable(), nullable); + assert_eq!( + field.data_type(), + &DataType::Timestamp( + TimeUnit::Microsecond, + Some(timezone.unwrap_or("UTC").into()), + ), + ); + } + } + Ok(()) + } +} diff --git a/crates/spark/src/function/implicit_cast.rs b/crates/spark/src/function/implicit_cast.rs index 3453eaf1..50ea0935 100644 --- a/crates/spark/src/function/implicit_cast.rs +++ b/crates/spark/src/function/implicit_cast.rs @@ -30,6 +30,10 @@ //! //! This module only chooses the target types. The cast that runs is DataFusion's, so how a //! malformed string or an overflowing value is cast is part of the CAST work in #88. +//! +//! For expressions that use only `ExpectsInputTypes`, construct the declarations with +//! [`SparkInputTypes::check_only`]. This shares the validation and error messages without +//! accepting implicit casts. Functions may then widen validated inputs for execution. use arrow::datatypes::{DataType, IntervalUnit}; use datafusion_common::config::ConfigOptions; @@ -78,12 +82,13 @@ pub(crate) enum SparkInputType { OneOf(Vec), } -/// The input types a function declares, and whether ANSI mode is on, which changes the casts -/// Spark allows. +/// The input types a function declares, whether implicit casts are allowed, and the ANSI +/// mode that controls those casts. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub(crate) struct SparkInputTypes { types: Vec, ansi_mode: bool, + implicit_casts: bool, signature: Signature, } @@ -93,10 +98,20 @@ impl SparkInputTypes { Self { types, ansi_mode: false, + implicit_casts: true, signature: Signature::user_defined(volatility), } } + /// Declares `ExpectsInputTypes`: validate inputs without implicit casts. + /// NULL uses the declared type's default, and dictionary layouts are unwrapped. + pub(crate) fn check_only(types: Vec, volatility: Volatility) -> Self { + Self { + implicit_casts: false, + ..Self::new(types, volatility) + } + } + /// The function's `Signature`, which leaves coercion to [`Self::coerce`]. pub(crate) fn signature(&self) -> &Signature { &self.signature @@ -114,7 +129,8 @@ impl SparkInputTypes { }) } - /// Casts each of `arg_types` to its declared type, for `ScalarUDFImpl::coerce_types`. + /// Chooses each argument's target type for `ScalarUDFImpl::coerce_types`. + /// In check-only mode, only accepted layouts and untyped NULLs are allowed. /// /// Returns Spark's `WRONG_NUM_ARGS` error for the wrong number of arguments, and /// `DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE` for an argument that cannot be cast. @@ -138,7 +154,19 @@ impl SparkInputTypes { .zip(&self.types) .enumerate() .map(|(idx, (arg_type, expected))| { - match implicit_cast(arg_type, expected, self.ansi_mode) { + let target = if self.implicit_casts { + implicit_cast(arg_type, expected, self.ansi_mode) + } else { + let layout = spark_layout(arg_type); + if expected.accepts(&layout) { + Some(layout) + } else if layout == DataType::Null { + expected.default_type() + } else { + None + } + }; + match target { Some(target) => Ok(target), None => plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The {} parameter of \ @@ -754,6 +782,69 @@ mod tests { ); } + #[test] + fn check_only_validates_without_implicit_casts() { + let types = SparkInputTypes::check_only(vec![T::Integral], Volatility::Stable); + for ansi_mode in [false, true] { + let mut config = ConfigOptions::default(); + config.execution.enable_ansi_mode = ansi_mode; + let types = types.with_config(&config).unwrap_or_else(|| types.clone()); + for input in [DataType::Int8, DataType::Int64, DataType::UInt32] { + assert_eq!( + types + .coerce("timestamp_micros", std::slice::from_ref(&input)) + .unwrap(), + vec![input] + ); + } + assert_eq!( + types.coerce("timestamp_micros", &[DataType::Null]).unwrap(), + vec![DataType::Int32] + ); + assert_eq!( + types + .coerce( + "timestamp_micros", + &[DataType::Dictionary( + Box::new(DataType::Int32), + Box::new(DataType::UInt32) + )] + ) + .unwrap(), + vec![DataType::UInt32] + ); + for input in [ + DataType::Utf8, + DataType::Float64, + DataType::Decimal128(10, 1), + DataType::Boolean, + ] { + let error = types + .coerce("timestamp_micros", &[input]) + .unwrap_err() + .strip_backtrace(); + assert!( + error.contains( + "The first parameter of `timestamp_micros` requires the \"INTEGRAL\" type" + ), + "{error}" + ); + } + assert!( + types + .coerce("timestamp_micros", &[]) + .unwrap_err() + .strip_backtrace() + .contains("WRONG_NUM_ARGS") + ); + } + // Concrete types demonstrate that validation does not cast a string or + // narrow an integer, unlike ImplicitCastInputTypes. + let types = SparkInputTypes::check_only(vec![T::Integer], Volatility::Immutable); + assert!(types.coerce("f", &[DataType::Utf8]).is_err()); + assert!(types.coerce("f", &[DataType::Int64]).is_err()); + } + #[test] fn with_config_follows_ansi_mode() { let types = SparkInputTypes::new(vec![T::Boolean], Volatility::Immutable); diff --git a/crates/sqllogictest/test_files/datetime/timestamp_micros.slt b/crates/sqllogictest/test_files/datetime/timestamp_micros.slt index 15a040e6..d5d39d6c 100644 --- a/crates/sqllogictest/test_files/datetime/timestamp_micros.slt +++ b/crates/sqllogictest/test_files/datetime/timestamp_micros.slt @@ -8,7 +8,134 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 -## Original Query: SELECT timestamp_micros(1230219000123123); -## PySpark 3.5.5 Result: {'timestamp_micros(1230219000123123)': datetime.datetime(2008, 12, 25, 7, 30, 0, 123123), 'typeof(timestamp_micros(1230219000123123))': 'timestamp', 'typeof(1230219000123123)': 'bigint'} -#query -#SELECT timestamp_micros(1230219000123123::bigint); +# Expected results verified against Apache Spark 4.2.0. + +# Microseconds are measured from the UTC epoch, including negative values. +query PPPP +SELECT timestamp_micros(0::BIGINT), timestamp_micros(-1::BIGINT), + timestamp_micros(1230219000123123::BIGINT), timestamp_micros(-1230219000123123::BIGINT); +---- +1970-01-01T00:00:00Z 1969-12-31T23:59:59.999999Z 2008-12-25T15:30:00.123123Z 1931-01-07T08:29:59.876877Z + +# All Spark signed integer types are accepted and widened without rescaling. +query PPPP +SELECT timestamp_micros(1::TINYINT) AS tiny, timestamp_micros(1::SMALLINT) AS small, + timestamp_micros(1::INT) AS integer_value, timestamp_micros(1::BIGINT) AS big; +---- +1970-01-01T00:00:00.000001Z 1970-01-01T00:00:00.000001Z 1970-01-01T00:00:00.000001Z 1970-01-01T00:00:00.000001Z + +query PP +SELECT timestamp_micros(NULL) AS untyped_null, timestamp_micros(NULL::BIGINT) AS typed_null; +---- +NULL NULL + +# Column input exercises physical execution in both SLT modes. +query PI +SELECT timestamp_micros(x), unix_micros(timestamp_micros(x)) +FROM (VALUES (-1::BIGINT), (0::BIGINT), (1::BIGINT), (NULL::BIGINT)) AS t(x); +---- +1969-12-31T23:59:59.999999Z -1 +1970-01-01T00:00:00Z 0 +1970-01-01T00:00:00.000001Z 1 +NULL NULL + +query PPP +SELECT timestamp_micros(a), timestamp_micros(b), timestamp_micros(c) +FROM (VALUES ((-128)::TINYINT, (-32768)::SMALLINT, (-2147483648)::INT)) AS t(a, b, c); +---- +1969-12-31T23:59:59.999872Z 1969-12-31T23:59:59.967232Z 1969-12-31T23:24:12.516352Z + +# Both ends of the i64 range are valid. Round-trip rather than format these +# timestamps, whose years exceed the calendar/formatter range. +query I +SELECT unix_micros(timestamp_micros(x)) +FROM (VALUES ((-9223372036854775808)::BIGINT), (9223372036854775807::BIGINT)) AS t(x); +---- +-9223372036854775808 +9223372036854775807 + +query I +SELECT unix_micros(timestamp_micros(arrow_cast(x, 'Dictionary(Int32, Int64)'))) +FROM (VALUES (-1::BIGINT), (1::BIGINT), (NULL::BIGINT)) AS t(x); +---- +-1 +1 +NULL + +# ExpectsInputTypes rejects non-integer arguments rather than casting them. +query error The first parameter of `timestamp_micros` requires the "INTEGRAL" type, however it has the type "DOUBLE". +SELECT timestamp_micros(1.0::DOUBLE); + +query error DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE +SELECT timestamp_micros(1.0::FLOAT); + +query error DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE +SELECT timestamp_micros(1.0::DECIMAL(10, 1)); + +query error DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE +SELECT timestamp_micros('1'); + +query error DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE +SELECT timestamp_micros(true); + +# ANSI mode does not change the accepted input types or range. +statement ok +SET datafusion.execution.enable_ansi_mode = true; + +query error DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE +SELECT timestamp_micros('1'); + +query II +SELECT unix_micros(timestamp_micros((-9223372036854775808)::BIGINT)), + unix_micros(timestamp_micros(9223372036854775807::BIGINT)); +---- +-9223372036854775808 9223372036854775807 + +statement ok +SET datafusion.execution.enable_ansi_mode = false; + +# Session time zone affects display, not the stored epoch count. +statement ok +SET datafusion.execution.time_zone = 'America/Los_Angeles'; + +query PP +SELECT timestamp_micros(1230219000123123::BIGINT), timestamp_micros(-1230219000123123::BIGINT); +---- +2008-12-25T07:30:00.123123-08:00 1931-01-07T00:29:59.876877-08:00 + +# Across the spring daylight saving transition, UTC values remain unchanged. +query PI +SELECT timestamp_micros(x), unix_micros(timestamp_micros(x)) +FROM (VALUES (1710064799999999::BIGINT), (1710064800000000::BIGINT)) AS t(x); +---- +2024-03-10T01:59:59.999999-08:00 1710064799999999 +2024-03-10T03:00:00-07:00 1710064800000000 + +statement ok +SET datafusion.execution.time_zone = 'UTC'; + +# Check the Arrow return type without relying on Spark's type display. +query T +SELECT arrow_typeof(timestamp_micros(0::BIGINT)); +---- +Timestamp(µs, "UTC") + +# Arrow unsigned types through UInt32 widen losslessly to BIGINT. +query III +SELECT unix_micros(timestamp_micros(arrow_cast(255, 'UInt8'))), + unix_micros(timestamp_micros(arrow_cast(65535, 'UInt16'))), + unix_micros(timestamp_micros(arrow_cast(4294967295, 'UInt32'))); +---- +255 65535 4294967295 + +query I +SELECT unix_micros(timestamp_micros(arrow_cast(x, 'Dictionary(Int32, UInt32)'))) +FROM (VALUES (0::BIGINT), (4294967295::BIGINT), (NULL::BIGINT)) AS t(x); +---- +0 +4294967295 +NULL + +# UInt64's full range cannot be represented by the timestamp's signed count. +query error DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE +SELECT timestamp_micros(arrow_cast(1, 'UInt64')); diff --git a/docs/functions/README.md b/docs/functions/README.md index efb17586..e8e68a9b 100644 --- a/docs/functions/README.md +++ b/docs/functions/README.md @@ -21,7 +21,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [Conditional Functions](scalar/conditional.md) (2) - [String Functions](scalar/string.md) (17) - [Binary String Functions](scalar/binary_string.md) (5) -- [Time and Date Functions](scalar/time_and_date.md) (27) +- [Time and Date Functions](scalar/time_and_date.md) (28) - [Array Functions](scalar/array.md) (10) - [Hashing Functions](scalar/hashing.md) (4) - [Other Functions](scalar/other.md) (6) @@ -137,6 +137,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [substr](scalar/string.md#substr) - [substring](scalar/string.md#substring) - [time_trunc](scalar/time_and_date.md#time_trunc) +- [timestamp_micros](scalar/time_and_date.md#timestamp_micros) - [to_binary](scalar/binary_string.md#to_binary) - [to_json](scalar/json.md#to_json) - [to_utc_timestamp](scalar/time_and_date.md#to_utc_timestamp) diff --git a/docs/functions/scalar/time_and_date.md b/docs/functions/scalar/time_and_date.md index 58a2c82a..82ccde3c 100644 --- a/docs/functions/scalar/time_and_date.md +++ b/docs/functions/scalar/time_and_date.md @@ -37,6 +37,7 @@ Part of the [datafusion-spark function reference](../README.md). - [next_day](#next_day) - [second](#second) - [time_trunc](#time_trunc) +- [timestamp_micros](#timestamp_micros) - [to_utc_timestamp](#to_utc_timestamp) - [trunc](#trunc) - [unix_date](#unix_date) @@ -575,6 +576,29 @@ time_trunc(unit, time) NULL ``` +## `timestamp_micros` + +Returns the timestamp that is `microseconds` microseconds after 1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike DataFusion's `to_timestamp_micros`, only integer types whose full range fits in BIGINT are accepted; floating-point, decimal and string inputs are rejected. The result has microsecond precision and the session time zone (`datafusion.execution.time_zone`, default UTC). The time zone affects display, not the stored microsecond count. Every BIGINT value is valid, including its minimum and maximum. Returns NULL for NULL input. + +```sql +timestamp_micros(microseconds) +``` + +### Arguments + +- **microseconds**: TINYINT, SMALLINT, INT or BIGINT count of microseconds since the UTC epoch. Arrow UInt8, UInt16 and UInt32 inputs are also accepted. UInt64 is rejected because its full range cannot fit in BIGINT. NULL returns NULL. + +### Example + +```sql +> SELECT timestamp_micros(0::BIGINT); +1970-01-01T00:00:00Z +> SELECT timestamp_micros(-1::BIGINT); +1969-12-31T23:59:59.999999Z +> SELECT timestamp_micros(NULL); +NULL +``` + ## `to_utc_timestamp` Treats `ts` as a wall-clock time in time zone `tz` and returns the UTC wall-clock time. This is the inverse of `from_utc_timestamp`. DataFusion has no built-in equivalent. A string `ts` is cast to a timestamp, so a string with an offset is first converted to UTC. The result has the same type as `ts`, including any time zone it carries. Returns NULL if either argument is NULL. Unlike `from_utc_timestamp`, `tz` is parsed by Arrow, so it accepts region IDs and `+HH:MM` offsets but not all of Spark's spellings (such as `GMT+1` or `PST`). An unknown zone is an error. Within a few hours of a daylight saving time change, the offset can be taken from the wrong side of the change, so the result can differ from Spark by the size of the change ([#110](https://github.com/apache/datafusion-spark/issues/110)).