From e3b90a920aec7ad908d9fb4add4ba853547d2bce Mon Sep 17 00:00:00 2001 From: Pastoray Date: Fri, 20 Nov 2026 22:57:31 +0100 Subject: [PATCH] feat: add spark-compatible is_variant_null --- crates/spark/src/function/mod.rs | 1 + .../src/function/variant/is_variant_null.rs | 420 ++++++++++++++++++ crates/spark/src/function/variant/mod.rs | 27 ++ crates/spark/src/lib.rs | 2 + .../test_files/variant/is_variant_null.slt | 152 +++++++ docs/functions/README.md | 5 +- docs/functions/scalar/time_and_date.md | 26 ++ docs/functions/scalar/variant.md | 40 ++ 8 files changed, 672 insertions(+), 1 deletion(-) create mode 100644 crates/spark/src/function/variant/is_variant_null.rs create mode 100644 crates/spark/src/function/variant/mod.rs create mode 100644 crates/sqllogictest/test_files/variant/is_variant_null.slt create mode 100644 docs/functions/scalar/variant.md diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 1c165bed..dde101e7 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -31,5 +31,6 @@ pub(crate) mod timezone_utils; pub(crate) mod type_coercion; pub mod url; pub mod utils; +pub mod variant; pub mod window; pub mod xml; diff --git a/crates/spark/src/function/variant/is_variant_null.rs b/crates/spark/src/function/variant/is_variant_null.rs new file mode 100644 index 00000000..1ffe3897 --- /dev/null +++ b/crates/spark/src/function/variant/is_variant_null.rs @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, StructArray}; +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, +}; +use datafusion_macros::user_doc; + +/// Spark-compatible `is_variant_null` expression. +/// +/// +#[user_doc( + doc_section(label = "Variant Functions"), + description = "Returns true only if the VARIANT value is a variant null, which is a JSON `null` stored in a \ + VARIANT, and false otherwise. A SQL NULL returns false, so the result is never NULL. \ + A VARIANT is the struct of the Arrow `arrow.parquet.variant` extension type, with a binary `metadata` field \ + and a binary `value` field, and the function reads the first byte of `value`. \ + An empty `value` raises `[MALFORMED_VARIANT]`. \ + Any other argument type is rejected at planning time, with no implicit casts, as in Spark. \ + Spark has had `is_variant_null` since 4.0.0 and has no VARIANT type before that.", + syntax_example = "is_variant_null(variant)", + sql_example = r#"```sql +> SELECT is_variant_null(named_struct('metadata', X'010000', 'value', X'00')); +true +> SELECT is_variant_null(named_struct('metadata', X'010000', 'value', X'0C0D')); +false +> SELECT is_variant_null(NULL); +false +```"#, + argument( + name = "variant", + description = "VARIANT expression: a struct with a binary `metadata` field and a binary `value` field. \ + A NULL returns false." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkIsVariantNull { + signature: Signature, +} + +impl Default for SparkIsVariantNull { + fn default() -> Self { + Self::new() + } +} + +impl SparkIsVariantNull { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkIsVariantNull { + fn name(&self) -> &str { + "is_variant_null" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + match arg_types { + // Spark accepts a NULL literal as a SQL NULL VARIANT + [data_type] if data_type == &DataType::Null || is_variant(data_type) => { + Ok(arg_types.to_vec()) + } + [other] => plan_err!( + "is_variant_null expects a VARIANT argument, a struct with binary \ + `metadata` and `value` fields, got {other}" + ), + _ => plan_err!( + "is_variant_null expects exactly one argument, got {}", + arg_types.len() + ), + } + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Boolean) + } + + fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { + // A SQL NULL input returns false, so the result is never NULL. + Ok(Arc::new(Field::new(self.name(), DataType::Boolean, false))) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let [input] = take_function_args(self.name(), args.args)?; + match input { + ColumnarValue::Array(array) => { + Ok(ColumnarValue::Array(Arc::new(is_variant_null(&array)?))) + } + ColumnarValue::Scalar(scalar) => { + let result = is_variant_null(&scalar.to_array()?)?; + Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some( + result.value(0), + )))) + } + } + } +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// Whether `data_type` is the struct of a VARIANT: binary `metadata` and +/// `value` fields. A shredded VARIANT has more fields, which are not read. +fn is_variant(data_type: &DataType) -> bool { + let DataType::Struct(fields) = data_type else { + return false; + }; + ["metadata", "value"].iter().all(|name| { + fields + .find(name) + .is_some_and(|(_, field)| is_binary(field.data_type())) + }) +} + +fn is_variant_null(array: &ArrayRef) -> Result { + if array.data_type() == &DataType::Null { + return Ok(BooleanArray::from(vec![false; array.len()])); + } + let Some(variants) = array.as_struct_opt() else { + return exec_err!( + "is_variant_null expects a VARIANT argument, got {}", + array.data_type() + ); + }; + let Some(values) = variants.column_by_name("value") else { + return exec_err!("is_variant_null expects a VARIANT with a `value` field"); + }; + match values.data_type() { + DataType::Binary => rows(variants, values.as_binary::().iter()), + DataType::LargeBinary => rows(variants, values.as_binary::().iter()), + DataType::BinaryView => rows(variants, values.as_binary_view().iter()), + other => exec_err!("is_variant_null expects a binary `value` field, got {other}"), + } +} + +fn rows<'a>( + variants: &StructArray, + values: impl Iterator>, +) -> Result { + values + .enumerate() + .map(|(row, value)| match value { + // A SQL NULL is not a variant null. A NULL `value` in a VARIANT that + // is not NULL holds no value either, or its value is in a shredded + // `typed_value` field. + _ if variants.is_null(row) => Ok(false), + None => Ok(false), + Some([]) => exec_err!( + "[MALFORMED_VARIANT] Variant binary is malformed. Please check the data source is valid. SQLSTATE: 22023" + ), + // The header byte of a variant null has basic type 0 (primitive) and + // type id 0 (null). + Some([header, ..]) => Ok(*header == 0), + }) + .collect::>>() + .map(BooleanArray::from) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{BinaryArray, BinaryViewArray, LargeBinaryArray}; + use arrow::buffer::NullBuffer; + use arrow::datatypes::Fields; + + const METADATA: &[u8] = &[0x01, 0x00, 0x00]; + + fn struct_of(fields: Vec<(&str, ArrayRef)>, nulls: Option>) -> StructArray { + let (names, arrays): (Vec<_>, Vec<_>) = fields.into_iter().unzip(); + let fields: Fields = names + .iter() + .zip(&arrays) + .map(|(name, array)| Field::new(*name, array.data_type().clone(), true)) + .collect::>() + .into(); + StructArray::new(fields, arrays, nulls.map(NullBuffer::from)) + } + + /// A VARIANT array whose rows have the given `value` bytes. + fn variants(values: &[Option<&[u8]>]) -> StructArray { + let metadata: Vec> = vec![Some(METADATA); values.len()]; + struct_of( + vec![ + ("metadata", Arc::new(BinaryArray::from(metadata))), + ("value", Arc::new(BinaryArray::from(values.to_vec()))), + ], + None, + ) + } + + fn evaluate(array: StructArray) -> Result { + let result = SparkIsVariantNull::new().invoke_with_args(ScalarFunctionArgs { + args: vec![ColumnarValue::Array(Arc::new(array.clone()))], + arg_fields: vec![Arc::new(Field::new("v", array.data_type().clone(), true))], + number_rows: array.len(), + return_field: Arc::new(Field::new("r", DataType::Boolean, false)), + config_options: Arc::new(Default::default()), + })?; + match result { + ColumnarValue::Array(array) => Ok(array.as_boolean().clone()), + ColumnarValue::Scalar(_) => panic!("expected an array"), + } + } + + #[test] + fn test_header_byte_decides() { + let result = evaluate(variants(&[ + Some(&[0x00]), // null + Some(&[0x04]), // true + Some(&[0x0C, 0x0D]), // int8 13 + Some(&[0x11, b'n', b'u', b'l', b'l']), // the string "null" + Some(&[0x03, 0x00, 0x00]), // [] + Some(&[0x02, 0x00, 0x00]), // {} + Some(&[0x00, 0x01, 0x02]), // only the first byte is read + ])) + .unwrap(); + assert_eq!( + result, + BooleanArray::from(vec![true, false, false, false, false, false, true]) + ); + } + + #[test] + fn test_sql_null_is_false() { + // A NULL struct row returns false, even with a `value` that would be a + // variant null or that is empty. + let array = struct_of( + vec![ + ( + "metadata", + Arc::new(BinaryArray::from(vec![Some(METADATA); 3])), + ), + ( + "value", + Arc::new(BinaryArray::from(vec![ + Some(&[0x00][..]), + Some(&[][..]), + Some(&[0x00][..]), + ])), + ), + ], + Some(vec![false, false, true]), + ); + assert_eq!( + evaluate(array).unwrap(), + BooleanArray::from(vec![false, false, true]) + ); + } + + #[test] + fn test_null_literal_is_false() { + let result = SparkIsVariantNull::new() + .invoke_with_args(ScalarFunctionArgs { + args: vec![ColumnarValue::Scalar(ScalarValue::Null)], + arg_fields: vec![Arc::new(Field::new("v", DataType::Null, true))], + number_rows: 1, + return_field: Arc::new(Field::new("r", DataType::Boolean, false)), + config_options: Arc::new(Default::default()), + }) + .unwrap(); + assert!(matches!( + result, + ColumnarValue::Scalar(ScalarValue::Boolean(Some(false))) + )); + } + + #[test] + fn test_null_value_is_false() { + let result = evaluate(variants(&[None, Some(&[0x00])])).unwrap(); + assert_eq!(result, BooleanArray::from(vec![false, true])); + } + + #[test] + fn test_empty_value_is_malformed() { + let err = evaluate(variants(&[Some(&[0x00]), Some(&[])])) + .unwrap_err() + .to_string(); + assert!(err.contains("[MALFORMED_VARIANT]"), "{err}"); + assert!(err.contains("SQLSTATE: 22023"), "{err}"); + } + + #[test] + fn test_binary_layouts_and_field_order() { + let values: Vec> = vec![Some(&[0x00]), Some(&[0x04])]; + let expected = BooleanArray::from(vec![true, false]); + let metadata: Vec> = vec![Some(METADATA); 2]; + + // `value` before `metadata`, as a large binary + let large = struct_of( + vec![ + ("value", Arc::new(LargeBinaryArray::from(values.clone()))), + ( + "metadata", + Arc::new(LargeBinaryArray::from(metadata.clone())), + ), + ], + None, + ); + assert_eq!(evaluate(large).unwrap(), expected); + + // a binary view, with a shredded `typed_value` field that is not read + let shredded = struct_of( + vec![ + ("metadata", Arc::new(BinaryViewArray::from(metadata))), + ("value", Arc::new(BinaryViewArray::from(values))), + ( + "typed_value", + Arc::new(arrow::array::Int32Array::from(vec![None, Some(1)])), + ), + ], + None, + ); + assert_eq!(evaluate(shredded).unwrap(), expected); + } + + #[test] + fn test_scalar_input() { + let array = variants(&[Some(&[0x00])]); + let func = SparkIsVariantNull::new(); + let result = func + .invoke_with_args(ScalarFunctionArgs { + args: vec![ColumnarValue::Scalar(ScalarValue::Struct(Arc::new( + array.clone(), + )))], + arg_fields: vec![Arc::new(Field::new( + "v", + array.data_type().clone(), + true, + ))], + number_rows: 1, + return_field: Arc::new(Field::new("r", DataType::Boolean, false)), + config_options: Arc::new(Default::default()), + }) + .unwrap(); + assert!(matches!( + result, + ColumnarValue::Scalar(ScalarValue::Boolean(Some(true))) + )); + } + + #[test] + fn test_result_is_not_nullable() { + let func = SparkIsVariantNull::new(); + let variant = variants(&[None]); + let arg_field = Arc::new(Field::new("v", variant.data_type().clone(), true)); + let field = func + .return_field_from_args(ReturnFieldArgs { + arg_fields: &[arg_field], + scalar_arguments: &[None], + }) + .unwrap(); + assert_eq!(field.data_type(), &DataType::Boolean); + assert!(!field.is_nullable()); + } + + #[test] + fn test_only_variant_is_accepted() { + let func = SparkIsVariantNull::new(); + let variant = variants(&[None]).data_type().clone(); + assert_eq!( + func.coerce_types(std::slice::from_ref(&variant)).unwrap(), + vec![variant.clone()] + ); + + let not_variants = [ + DataType::Binary, + DataType::Utf8, + DataType::Boolean, + // a struct without a binary `value` field + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, true), + Field::new("value", DataType::Utf8, true), + ])), + DataType::Struct(Fields::from(vec![Field::new( + "value", + DataType::Binary, + true, + )])), + ]; + for data_type in not_variants { + let err = func + .coerce_types(std::slice::from_ref(&data_type)) + .unwrap_err(); + assert!( + err.to_string().contains("expects a VARIANT argument"), + "{data_type}: {err}" + ); + } + // a NULL literal is a SQL NULL VARIANT, as in Spark + assert_eq!( + func.coerce_types(&[DataType::Null]).unwrap(), + vec![DataType::Null] + ); + assert!(func.coerce_types(&[]).is_err()); + assert!(func.coerce_types(&[variant.clone(), variant]).is_err()); + } +} diff --git a/crates/spark/src/function/variant/mod.rs b/crates/spark/src/function/variant/mod.rs new file mode 100644 index 00000000..42883693 --- /dev/null +++ b/crates/spark/src/function/variant/mod.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark VARIANT functions, such as `is_variant_null`. + +pub mod is_variant_null; + +use datafusion_expr::ScalarUDF; +use datafusion_functions::make_udf_function; +use std::sync::Arc; + +make_udf_function!(is_variant_null::SparkIsVariantNull, is_variant_null); + +pub mod expr_fn { + use datafusion_functions::export_functions; + + export_functions!(( + is_variant_null, + "Returns true only if the VARIANT value is a variant null (a JSON `null`), and false otherwise, including for a SQL NULL.", + arg1 + )); +} + +pub fn functions() -> Vec> { + vec![is_variant_null()] +} diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 97cc28fb..c1ef22ba 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -219,6 +219,7 @@ pub mod expr_fn { pub use super::function::r#struct::expr_fn::*; pub use super::function::table::expr_fn::*; pub use super::function::url::expr_fn::*; + pub use super::function::variant::expr_fn::*; pub use super::function::window::expr_fn::*; pub use super::function::xml::expr_fn::*; } @@ -245,6 +246,7 @@ pub fn all_default_scalar_functions() -> Vec> { .chain(function::string::functions()) .chain(function::r#struct::functions()) .chain(function::url::functions()) + .chain(function::variant::functions()) .chain(function::xml::functions()) .collect::>() } diff --git a/crates/sqllogictest/test_files/variant/is_variant_null.slt b/crates/sqllogictest/test_files/variant/is_variant_null.slt new file mode 100644 index 00000000..50716c69 --- /dev/null +++ b/crates/sqllogictest/test_files/variant/is_variant_null.slt @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible is_variant_null +# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#is_variant_null +# Spark 4.0.0 added it, and it is the same in 4.0.4, 4.1.3 and 4.2.0. +# +# This crate has no `parse_json` yet, so a VARIANT is built as the struct of the +# Arrow `arrow.parquet.variant` type: `named_struct('metadata', ..., 'value', ...)`, +# with a minimal valid metadata (version 1, an empty dictionary) of X'010000'. +# Spark has no `named_struct` that is a VARIANT, so the records that build one are +# marked `skipif spark`. The expected values were checked on Spark 4.2.0 with +# `parse_json`, for example `is_variant_null(parse_json('null'))` is true and +# `is_variant_null(parse_json('13'))` is false. +# +# The `value` bytes below are Spark's variant encoding. The first byte is the header: +# X'00' null +# X'04' / X'08' true / false +# X'0C0D' the 8-bit integer 13 +# X'01' the empty string +# X'116E756C6C' the string "null", which is not a variant null +# X'030000' the empty array [] +# X'03020001020000' the array [null, null] +# X'020000' the empty object {} + +# A NULL literal is a SQL NULL, which is not a variant null (and the result is not NULL) +query B +SELECT is_variant_null(NULL); +---- +false + +# A variant null is a JSON null stored in a VARIANT +skipif spark +query B +SELECT is_variant_null(named_struct('metadata', X'010000', 'value', X'00')); +---- +true + +# Every other value is false: strings, numbers, booleans, arrays and objects +skipif spark +query BBBBBBBBB +SELECT + is_variant_null(named_struct('metadata', X'010000', 'value', X'116E756C6C')), + is_variant_null(named_struct('metadata', X'010000', 'value', X'01')), + is_variant_null(named_struct('metadata', X'010000', 'value', X'0C0D')), + is_variant_null(named_struct('metadata', X'010000', 'value', X'0C00')), + is_variant_null(named_struct('metadata', X'010000', 'value', X'04')), + is_variant_null(named_struct('metadata', X'010000', 'value', X'08')), + is_variant_null(named_struct('metadata', X'010000', 'value', X'030000')), + is_variant_null(named_struct('metadata', X'010000', 'value', X'03020001020000')), + is_variant_null(named_struct('metadata', X'010000', 'value', X'020000')); +---- +false false false false false false false false false + +# The metadata is ignored +skipif spark +query BB +SELECT + is_variant_null(named_struct('metadata', X'', 'value', X'00')), + is_variant_null(named_struct('metadata', X'0101000103616263', 'value', X'0C0D')); +---- +true false + +# Column input, with a SQL NULL VARIANT and a variant null in the same column +skipif spark +query IB +SELECT i, is_variant_null(v) FROM ( + SELECT i, CASE WHEN i = 3 THEN NULL + WHEN i = 1 THEN named_struct('metadata', X'010000', 'value', X'00') + ELSE named_struct('metadata', X'010000', 'value', X'0C0D') END AS v + FROM (VALUES (1), (2), (3)) AS t(i) +) ORDER BY i; +---- +1 true +2 false +3 false + +# A NULL `value` in a VARIANT that is not NULL holds no value, which is false +skipif spark +query B +SELECT is_variant_null(named_struct('metadata', X'010000', 'value', arrow_cast(NULL, 'Binary'))); +---- +false + +# An empty `value` is malformed +skipif spark +query error \[MALFORMED_VARIANT\] Variant binary is malformed\. Please check the data source is valid\. SQLSTATE: 22023 +SELECT is_variant_null(named_struct('metadata', X'010000', 'value', X'')); + +# ... also in a column, where it raises for the whole query even next to valid rows +skipif spark +query error \[MALFORMED_VARIANT\] +SELECT is_variant_null(v) FROM ( + SELECT CASE WHEN i = 1 THEN named_struct('metadata', X'010000', 'value', X'00') + ELSE named_struct('metadata', X'010000', 'value', X'') END AS v + FROM (VALUES (1), (2)) AS t(i) +); + +# An empty `value` in a SQL NULL VARIANT is not read +skipif spark +query B +SELECT is_variant_null(CASE WHEN i = 1 THEN NULL ELSE named_struct('metadata', X'010000', 'value', X'') END) +FROM (VALUES (1)) AS t(i); +---- +false + +# Large binary and binary view fields give the same result +skipif spark +query BB +SELECT + is_variant_null(named_struct('metadata', arrow_cast(X'010000', 'LargeBinary'), 'value', arrow_cast(X'00', 'LargeBinary'))), + is_variant_null(named_struct('metadata', arrow_cast(X'010000', 'BinaryView'), 'value', arrow_cast(X'0C0D', 'BinaryView'))); +---- +true false + +# The result does not depend on ANSI mode +statement ok +set datafusion.execution.enable_ansi_mode = true; + +skipif spark +query B +SELECT is_variant_null(named_struct('metadata', X'010000', 'value', X'00')); +---- +true + +query B +SELECT is_variant_null(NULL); +---- +false + +statement ok +set datafusion.execution.enable_ansi_mode = false; + +# Only a VARIANT is accepted, with no implicit casts. Spark rejects these types too. +statement error expects a VARIANT argument +SELECT is_variant_null('null'); + +statement error expects a VARIANT argument +SELECT is_variant_null(1); + +statement error expects a VARIANT argument +SELECT is_variant_null(X'00'); + +statement error expects a VARIANT argument +SELECT is_variant_null(named_struct('metadata', X'010000', 'value', 'null')); + +statement error expects a VARIANT argument +SELECT is_variant_null(named_struct('value', X'00')); + +statement error expects exactly one argument +SELECT is_variant_null(NULL, NULL); diff --git a/docs/functions/README.md b/docs/functions/README.md index e8e68a9b..fa077b48 100644 --- a/docs/functions/README.md +++ b/docs/functions/README.md @@ -21,12 +21,13 @@ function differs from the DataFusion built-in of the same name, if there is one. - [Conditional Functions](scalar/conditional.md) (2) - [String Functions](scalar/string.md) (17) - [Binary String Functions](scalar/binary_string.md) (5) -- [Time and Date Functions](scalar/time_and_date.md) (28) +- [Time and Date Functions](scalar/time_and_date.md) (29) - [Array Functions](scalar/array.md) (10) - [Hashing Functions](scalar/hashing.md) (4) - [Other Functions](scalar/other.md) (6) - [JSON Functions](scalar/json.md) (3) - [URL Functions](scalar/url.md) (3) +- [Variant Functions](scalar/variant.md) (1) ## Aggregate Functions @@ -77,6 +78,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [datediff](scalar/time_and_date.md#datediff) - [datepart](scalar/time_and_date.md#datepart) - [dayname](scalar/time_and_date.md#dayname) +- [dayofweek](scalar/time_and_date.md#dayofweek) - [elt](scalar/string.md#elt) - [encode](scalar/binary_string.md#encode) - [equal_null](scalar/other.md#equal_null) @@ -94,6 +96,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [if](scalar/conditional.md#if) - [ilike](scalar/string.md#ilike) - [is_valid_utf8](scalar/string.md#is_valid_utf8) +- [is_variant_null](scalar/variant.md#is_variant_null) - [json_array_length](scalar/json.md#json_array_length) - [json_tuple](scalar/json.md#json_tuple) - [last_day](scalar/time_and_date.md#last_day) diff --git a/docs/functions/scalar/time_and_date.md b/docs/functions/scalar/time_and_date.md index 82ccde3c..20cd32ed 100644 --- a/docs/functions/scalar/time_and_date.md +++ b/docs/functions/scalar/time_and_date.md @@ -25,6 +25,7 @@ Part of the [datafusion-spark function reference](../README.md). - [datediff](#datediff) - [datepart](#datepart) - [dayname](#dayname) +- [dayofweek](#dayofweek) - [from_utc_timestamp](#from_utc_timestamp) - [hour](#hour) - [last_day](#last_day) @@ -284,6 +285,31 @@ Mon NULL ``` +## `dayofweek` + +Returns the day of the week as an INT from 1 (Sunday) to 7 (Saturday). This differs from `weekday`, which counts from 0 (Monday) to 6 (Sunday), and from DataFusion's `date_part('dow', ...)`, which counts from 0 (Sunday) to 6 (Saturday). DataFusion has no built-in `dayofweek`. Strings are implicitly cast to DATE. A malformed string is an error, even with ANSI mode off where Spark returns NULL. A string that Spark reads as a partial date, such as '2009' or '2009-07', is also an error, where Spark uses the first day of that year or month. Timestamps are cast to DATE first, so the time of day is ignored. A timestamp with a time zone is converted in its own time zone, not the session time zone that Spark uses. Returns NULL if the input is NULL. + +```sql +dayofweek(date) +``` + +### Arguments + +- **date**: DATE, TIMESTAMP or string expression. NULL returns NULL. + +### Example + +```sql +> SELECT dayofweek('2009-07-30'::DATE); +5 +> SELECT dayofweek('2009-07-30'); +5 +> SELECT dayofweek(arrow_cast('2024-01-07 23:59:59', 'Timestamp(Microsecond, None)')); +1 +> SELECT dayofweek(NULL::DATE); +NULL +``` + ## `from_utc_timestamp` Treats `ts` as a UTC wall-clock time and returns the wall-clock time in time zone `tz`. DataFusion has no built-in equivalent. A string `ts` is cast to a timestamp, so a string with an offset is first converted to UTC. The result has the same type as `ts`, including any time zone it carries. Returns NULL if either argument is NULL. `tz` accepts Spark's spellings: region IDs, offsets such as `+01:00`, `GMT+1`, `UTC+1:2` and `UT-010203`, Java short IDs such as `PST`, and legacy `SystemV/` IDs. An offset beyond 18 hours or an unknown zone is an error. Differences from Spark: `tz` is only parsed for rows where both arguments are non-NULL, so an invalid constant `tz` can return NULL where Spark raises an error; conversion is limited to Chrono's calendar range, which is narrower than the range Spark implements but covers Spark's documented range; and region rules from `chrono-tz` may differ after 2099, because it does not extrapolate later daylight saving time transitions. diff --git a/docs/functions/scalar/variant.md b/docs/functions/scalar/variant.md new file mode 100644 index 00000000..a8259d73 --- /dev/null +++ b/docs/functions/scalar/variant.md @@ -0,0 +1,40 @@ + + + + +# Variant Functions (scalar functions) + +Part of the [datafusion-spark function reference](../README.md). + +- [is_variant_null](#is_variant_null) + +## `is_variant_null` + +Returns true only if the VARIANT value is a variant null, which is a JSON `null` stored in a VARIANT, and false otherwise. A SQL NULL returns false, so the result is never NULL. A VARIANT is the struct of the Arrow `arrow.parquet.variant` extension type, with a binary `metadata` field and a binary `value` field, and the function reads the first byte of `value`. An empty `value` raises `[MALFORMED_VARIANT]`. Any other argument type is rejected at planning time, with no implicit casts, as in Spark. Spark has had `is_variant_null` since 4.0.0 and has no VARIANT type before that. + +```sql +is_variant_null(variant) +``` + +### Arguments + +- **variant**: VARIANT expression: a struct with a binary `metadata` field and a binary `value` field. A NULL returns false. + +### Example + +```sql +> SELECT is_variant_null(named_struct('metadata', X'010000', 'value', X'00')); +true +> SELECT is_variant_null(named_struct('metadata', X'010000', 'value', X'0C0D')); +false +> SELECT is_variant_null(NULL); +false +```