Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ details.
`inputTypes` with `SparkInputTypes` (`function/implicit_cast.rs`), which supplies `signature()`,
`coerce_types()` and `with_updated_config()` with Spark's errors and ANSI rules. See `sha2`. It
only chooses the target types, so the cast that then runs is not yet Spark's.
- **Checked input types.** When the expression uses `ExpectsInputTypes` without implicit casts,
use `SparkInputTypes::check_only`. It shares the declared-type validation and Spark errors,
accepts untyped NULLs, and unwraps dictionaries. Widen accepted inputs separately when the
execution kernel needs a common representation. See `timestamp_micros`.
- **Common type of several arguments.** When Spark finds one type for several arguments, as
`array_position` does, call the rules in `function/type_coercion.rs` (for example
`tightest_common_type()`) from `coerce_types()`.
Expand Down
8 changes: 8 additions & 0 deletions crates/spark/src/function/datetime/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ pub mod make_time;
pub mod monthname;
pub mod next_day;
pub mod time_trunc;
pub mod timestamp_micros;
pub(crate) mod timezone;
pub mod to_utc_timestamp;
pub mod trunc;
Expand Down Expand Up @@ -60,6 +61,7 @@ make_udf_function!(make_time::SparkMakeTime, make_time);
make_udf_function!(monthname::SparkMonthName, monthname);
make_udf_function!(next_day::SparkNextDay, next_day);
make_udf_function!(time_trunc::SparkTimeTrunc, time_trunc);
make_udf_function!(timestamp_micros::SparkTimestampMicros, timestamp_micros);
make_udf_function!(to_utc_timestamp::SparkToUtcTimestamp, to_utc_timestamp);
make_udf_function!(trunc::SparkTrunc, trunc);
make_udf_function!(unix::SparkUnixDate, unix_date);
Expand Down Expand Up @@ -186,6 +188,11 @@ pub mod expr_fn {
"Interpret a given timestamp `ts` in UTC timezone and then convert it to timezone `tz`.",
ts tz
));
export_functions!((
timestamp_micros,
"Creates a timestamp from microseconds since the UTC epoch.",
microseconds
));
export_functions!((
to_utc_timestamp,
"Interpret a given timestamp `ts` in timezone `tz` and then convert it to UTC timezone.",
Expand Down Expand Up @@ -240,6 +247,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
next_day(),
second(),
time_trunc(),
timestamp_micros(),
to_utc_timestamp(),
trunc(),
unix_date(),
Expand Down
161 changes: 161 additions & 0 deletions crates/spark/src/function/datetime/timestamp_micros.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use std::sync::Arc;

use arrow::datatypes::{DataType, Field, FieldRef, TimeUnit};
use datafusion_common::config::ConfigOptions;
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err, plan_err};
use datafusion_expr::{
ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,
ScalarUDFImpl, Signature, Volatility,
};
use datafusion_macros::user_doc;

use crate::function::implicit_cast::{SparkInputType, SparkInputTypes};

/// Spark-compatible `timestamp_micros` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#timestamp_micros>
#[user_doc(
doc_section(label = "Time and Date Functions"),
description = "Returns the timestamp that is `microseconds` microseconds after \
1970-01-01 00:00:00 UTC. Negative values precede the epoch. Unlike DataFusion's \
`to_timestamp_micros`, only integer types whose full range fits in BIGINT are \
accepted; floating-point, decimal and string inputs are rejected. The result has microsecond precision \
and the session time zone (`datafusion.execution.time_zone`, default UTC). \
The time zone affects display, not the stored microsecond count. Every BIGINT \
value is valid, including its minimum and maximum. Returns NULL for NULL input.",
syntax_example = "timestamp_micros(microseconds)",
sql_example = r#"```sql
> SELECT timestamp_micros(0::BIGINT);
1970-01-01T00:00:00Z
> SELECT timestamp_micros(-1::BIGINT);
1969-12-31T23:59:59.999999Z
> SELECT timestamp_micros(NULL);
NULL
```"#,
argument(
name = "microseconds",
description = "TINYINT, SMALLINT, INT or BIGINT count of microseconds since the UTC epoch. Arrow UInt8, UInt16 and UInt32 inputs are also accepted. UInt64 is rejected because its full range cannot fit in BIGINT. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkTimestampMicros {
input_types: SparkInputTypes,
timezone: Arc<str>,
}

impl Default for SparkTimestampMicros {
fn default() -> Self {
Self::new()
}
}

impl SparkTimestampMicros {
pub fn new() -> Self {
Self::new_with_config(&ConfigOptions::default())
}

pub fn new_with_config(config: &ConfigOptions) -> Self {
Self {
input_types: SparkInputTypes::check_only(
vec![SparkInputType::Integral],
Volatility::Stable,
),
timezone: config
.execution
.time_zone
.as_ref()
.map_or("UTC", |tz| tz.as_str())
.into(),
}
}
}

impl ScalarUDFImpl for SparkTimestampMicros {
fn name(&self) -> &str {
"timestamp_micros"
}

fn signature(&self) -> &Signature {
self.input_types.signature()
}

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree SparkInputTypes isn't a fit as it stands, since it models ImplicitCastInputTypes and would cast '1'. timestamp_millis and timestamp_seconds will need the same ExpectsInputTypes treatment soon, though. What do you think about adding a check-only mode to implicit_cast.rs so all three share it? That would also give this error the same Spark-shaped message the other functions use ("The first parameter of timestamp_micros requires the "INTEGRAL" type..."). If you'd rather keep this PR focused, a tracking issue linked here would work too.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Implemented in 1526fc0. Added SparkInputTypes::check_only in implicit_cast.rs and switched timestamp_micros to use it. It validates the declared input types without implicit casts, handles NULLs and dictionary layouts, and shares the Spark-style argument errors. The helper is available for timestamp_millis and timestamp_seconds when those are implemented. Added coverage for validation in both ANSI modes and the error message. Workspace Clippy and all 516 SQL test runs pass.

let types = self.input_types.coerce(self.name(), arg_types)?;
if types[0] == DataType::UInt64 {
return plan_err!(
"[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of \
`timestamp_micros` requires an integer type whose full range fits in \
BIGINT, however it has the type \"UInt64\". SQLSTATE: 42K09"
);
}
Ok(vec![DataType::Int64])
}

fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
internal_err!("return_field_from_args should be used instead")
}

fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
let [input] = take_function_args(self.name(), args.arg_fields)?;
Ok(Arc::new(Field::new(
self.name(),
DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::clone(&self.timezone))),
input.is_nullable(),
)))
}

fn with_updated_config(&self, config: &ConfigOptions) -> Option<ScalarUDF> {
Some(ScalarUDF::from(Self::new_with_config(config)))
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let [microseconds] = take_function_args(self.name(), args.args)?;
// Both types store an i64 count, so this does not scale the value or
// convert it through a calendar representation (which has a smaller range).
microseconds.cast_to(
&DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::clone(&self.timezone))),
None,
)
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn return_field_tracks_session_timezone_and_input_nullability() -> Result<()> {
for timezone in [None, Some("America/Los_Angeles")] {
let mut config = ConfigOptions::default();
config.execution.time_zone = timezone.map(str::parse).transpose()?;
let function = SparkTimestampMicros::new()
.with_updated_config(&config)
.unwrap();
for nullable in [false, true] {
let input = Arc::new(Field::new("micros", DataType::Int64, nullable));
let field = function.inner().return_field_from_args(ReturnFieldArgs {
arg_fields: &[input],
scalar_arguments: &[None],
})?;
assert_eq!(field.is_nullable(), nullable);
assert_eq!(
field.data_type(),
&DataType::Timestamp(
TimeUnit::Microsecond,
Some(timezone.unwrap_or("UTC").into()),
),
);
}
}
Ok(())
}
}
99 changes: 95 additions & 4 deletions crates/spark/src/function/implicit_cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@
//!
//! This module only chooses the target types. The cast that runs is DataFusion's, so how a
//! malformed string or an overflowing value is cast is part of the CAST work in #88.
//!
//! For expressions that use only `ExpectsInputTypes`, construct the declarations with
//! [`SparkInputTypes::check_only`]. This shares the validation and error messages without
//! accepting implicit casts. Functions may then widen validated inputs for execution.

use arrow::datatypes::{DataType, IntervalUnit};
use datafusion_common::config::ConfigOptions;
Expand Down Expand Up @@ -78,12 +82,13 @@ pub(crate) enum SparkInputType {
OneOf(Vec<SparkInputType>),
}

/// The input types a function declares, and whether ANSI mode is on, which changes the casts
/// Spark allows.
/// The input types a function declares, whether implicit casts are allowed, and the ANSI
/// mode that controls those casts.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub(crate) struct SparkInputTypes {
types: Vec<SparkInputType>,
ansi_mode: bool,
implicit_casts: bool,
signature: Signature,
}

Expand All @@ -93,10 +98,20 @@ impl SparkInputTypes {
Self {
types,
ansi_mode: false,
implicit_casts: true,
signature: Signature::user_defined(volatility),
}
}

/// Declares `ExpectsInputTypes`: validate inputs without implicit casts.
/// NULL uses the declared type's default, and dictionary layouts are unwrapped.
pub(crate) fn check_only(types: Vec<SparkInputType>, volatility: Volatility) -> Self {
Self {
implicit_casts: false,
..Self::new(types, volatility)
}
}

/// The function's `Signature`, which leaves coercion to [`Self::coerce`].
pub(crate) fn signature(&self) -> &Signature {
&self.signature
Expand All @@ -114,7 +129,8 @@ impl SparkInputTypes {
})
}

/// Casts each of `arg_types` to its declared type, for `ScalarUDFImpl::coerce_types`.
/// Chooses each argument's target type for `ScalarUDFImpl::coerce_types`.
/// In check-only mode, only accepted layouts and untyped NULLs are allowed.
///
/// Returns Spark's `WRONG_NUM_ARGS` error for the wrong number of arguments, and
/// `DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE` for an argument that cannot be cast.
Expand All @@ -138,7 +154,19 @@ impl SparkInputTypes {
.zip(&self.types)
.enumerate()
.map(|(idx, (arg_type, expected))| {
match implicit_cast(arg_type, expected, self.ansi_mode) {
let target = if self.implicit_casts {
implicit_cast(arg_type, expected, self.ansi_mode)
} else {
let layout = spark_layout(arg_type);
if expected.accepts(&layout) {
Some(layout)
} else if layout == DataType::Null {
expected.default_type()
} else {
None
}
};
match target {
Some(target) => Ok(target),
None => plan_err!(
"[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The {} parameter of \
Expand Down Expand Up @@ -754,6 +782,69 @@ mod tests {
);
}

#[test]
fn check_only_validates_without_implicit_casts() {
let types = SparkInputTypes::check_only(vec![T::Integral], Volatility::Stable);
for ansi_mode in [false, true] {
let mut config = ConfigOptions::default();
config.execution.enable_ansi_mode = ansi_mode;
let types = types.with_config(&config).unwrap_or_else(|| types.clone());
for input in [DataType::Int8, DataType::Int64, DataType::UInt32] {
assert_eq!(
types
.coerce("timestamp_micros", std::slice::from_ref(&input))
.unwrap(),
vec![input]
);
}
assert_eq!(
types.coerce("timestamp_micros", &[DataType::Null]).unwrap(),
vec![DataType::Int32]
);
assert_eq!(
types
.coerce(
"timestamp_micros",
&[DataType::Dictionary(
Box::new(DataType::Int32),
Box::new(DataType::UInt32)
)]
)
.unwrap(),
vec![DataType::UInt32]
);
for input in [
DataType::Utf8,
DataType::Float64,
DataType::Decimal128(10, 1),
DataType::Boolean,
] {
let error = types
.coerce("timestamp_micros", &[input])
.unwrap_err()
.strip_backtrace();
assert!(
error.contains(
"The first parameter of `timestamp_micros` requires the \"INTEGRAL\" type"
),
"{error}"
);
}
assert!(
types
.coerce("timestamp_micros", &[])
.unwrap_err()
.strip_backtrace()
.contains("WRONG_NUM_ARGS")
);
}
// Concrete types demonstrate that validation does not cast a string or
// narrow an integer, unlike ImplicitCastInputTypes.
let types = SparkInputTypes::check_only(vec![T::Integer], Volatility::Immutable);
assert!(types.coerce("f", &[DataType::Utf8]).is_err());
assert!(types.coerce("f", &[DataType::Int64]).is_err());
}

#[test]
fn with_config_follows_ansi_mode() {
let types = SparkInputTypes::new(vec![T::Boolean], Volatility::Immutable);
Expand Down
Loading
Loading