diff --git a/Cargo.lock b/Cargo.lock index 05a58074..717edb0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1614,6 +1614,8 @@ dependencies = [ "num-traits", "percent-encoding", "rand 0.9.5", + "regex", + "regex-syntax", "serde", "serde_json", "sha1", diff --git a/Cargo.toml b/Cargo.toml index e02a721b..a0e6da33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,8 @@ datafusion-session = { git = "https://github.com/apache/datafusion", branch = "b log = "^0.4" num-traits = { version = "0.2" } rand = "0.9" +regex = "1" +regex-syntax = "0.8" serde = "1" serde_json = "1" sha2 = "^0.11.0" diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index 49f9d467..a267baa8 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -54,6 +54,8 @@ log = { workspace = true } num-traits = { workspace = true } percent-encoding = "2.3.2" rand = { workspace = true } +regex = { workspace = true } +regex-syntax = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha1 = "0.11" diff --git a/crates/spark/src/function/string/mod.rs b/crates/spark/src/function/string/mod.rs index 7530ec39..ae4ff80e 100644 --- a/crates/spark/src/function/string/mod.rs +++ b/crates/spark/src/function/string/mod.rs @@ -20,6 +20,7 @@ pub mod like; pub mod luhn_check; pub mod make_valid_utf8; pub mod quote; +pub mod regexp_extract; pub mod soundex; pub mod space; pub mod substring; @@ -48,6 +49,7 @@ make_udf_function!(soundex::SparkSoundex, soundex); make_udf_function!(make_valid_utf8::SparkMakeValidUtf8, make_valid_utf8); make_udf_function!(is_valid_utf8::SparkIsValidUtf8, is_valid_utf8); make_udf_function!(quote::SparkQuote, quote); +make_udf_function!(regexp_extract::SparkRegexpExtract, regexp_extract); make_udf_function!(to_binary::SparkToBinary, to_binary); make_udf_function!(to_binary::SparkTryToBinary, try_to_binary); @@ -141,6 +143,11 @@ pub mod expr_fn { "Returns str enclosed by single quotes and each instance of single quote in it is preceded by a backslash", str )); + export_functions!(( + regexp_extract, + "Extracts the group `idx` (default 1, 0 for the whole match) of the first match of `regexp` in `str`.", + str regexp idx + )); export_functions!(( to_binary, "Converts the input str to a binary value based on the supplied fmt, which must be a case-insensitive literal of 'hex', 'utf-8', 'utf8' or 'base64'. Defaults to 'hex'.", @@ -174,6 +181,7 @@ pub fn functions() -> Vec> { make_valid_utf8(), is_valid_utf8(), quote(), + regexp_extract(), to_binary(), try_to_binary(), ] diff --git a/crates/spark/src/function/string/regexp_extract.rs b/crates/spark/src/function/string/regexp_extract.rs new file mode 100644 index 00000000..14bb8dbe --- /dev/null +++ b/crates/spark/src/function/string/regexp_extract.rs @@ -0,0 +1,778 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::sync::{Arc, Mutex}; + +use arrow::array::{ + Array, ArrayRef, AsArray, Int32Array, StringArray, StringArrayType, StringBuilder, + new_null_array, +}; +use arrow::compute::{can_cast_types, cast}; +use arrow::datatypes::DataType; +use datafusion_common::cast::as_int32_array; +use datafusion_common::utils::offset_span_len; +use datafusion_common::{ + DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, + internal_datafusion_err, internal_err, plan_err, +}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; +use datafusion_macros::user_doc; +use regex::{CaptureLocations, Regex}; +use regex_syntax::ast::{ + AssertionKind, Ast, ClassSetBinaryOp, ClassSetBinaryOpKind, ClassSetItem, + ClassUnicode, ClassUnicodeKind, Flag, Flags, FlagsItemKind, GroupKind, Visitor, +}; + +use crate::function::error_utils::invalid_arg_count_exec_err; + +const NAME: &str = "regexp_extract"; + +/// Spark-compatible `regexp_extract` expression. +/// +/// +#[user_doc( + doc_section(label = "Regular Expression Functions"), + description = "Returns the text matched by group `idx` of the first match of `regexp` in `str`. \ + Unlike DataFusion's `regexp_match`, which returns a list of every capture group, this returns a single string. \ + Returns an empty string if there is no match, or if the group did not take part in the match. \ + `idx` is only checked when the pattern matches: an `idx` outside `0..=groupCount` is an error \ + (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and later; Spark 3.4 raised \ + `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty string on a row that does not. \ + An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). Returns NULL if any argument is NULL, \ + and a NULL row is not checked, so an invalid pattern is only reported on a row it is applied to. \ + As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \ + Known differences from Spark (https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \ + `regex` engine, not Java's `java.util.regex`. Look-around, backreferences, quoting with `\\Q...\\E`, \ + `\\Z`, `\\h`, atomic groups `(?>...)`, the `(?d)` flag, and patterns that exceed the Rust engine's \ + compiled-size limit raise an \"unsupported regex pattern\" error, where Spark evaluates them. Only unclosed \ + or unopened groups and classes, invalid class ranges and repetitions, and unknown flags are reported as \ + `INVALID_PARAMETER_VALUE.PATTERN`; any other pattern the Rust engine rejects is reported as unsupported, \ + including a few that Java rejects too, such as `\\y`. \ + Rust syntax that Java rejects is also `INVALID_PARAMETER_VALUE.PATTERN`: the `R` flag, `\\b{start}` and \ + the other `\\b{...}` word boundaries, `(?P...)`, and group names that are not an ASCII letter \ + followed by ASCII letters and digits. Syntax that both engines accept but read differently is reported as \ + unsupported rather than evaluated to a different answer: the `U` flag (Java's UNICODE_CHARACTER_CLASS, \ + which swaps greediness in Rust), `[:alpha:]` and the other POSIX classes in brackets, the class operators \ + `--` and `~~`, `\\<` and `\\>`, and the names of Java's ASCII-only POSIX classes, such as `\\p{Alpha}` \ + and `\\p{Lower}`. Other Unicode property names that Java does not know, such as `\\p{Letter}` or \ + `\\p{Latin}`, are accepted, where Java rejects them. \ + Possessive quantifiers are read as nested greedy repetition (`a++` as `(a+)+`), so they can match \ + where Java's would not. \ + `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match only ASCII. \ + `.` matches `\\r`, where Java's does not. `$` matches only at the very end of the input, where Java's also \ + matches before a final line terminator. `(?i)` folds case using Unicode rules (so `k` matches the Kelvin \ + sign), where Java folds only ASCII by default.", + syntax_example = "regexp_extract(str, regexp[, idx])", + sql_example = r#"```sql +> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)'); +1 +> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0); +1a +> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2); +a +> SELECT regexp_extract('abc', '(\d+)', 1); +-- empty string +> SELECT regexp_extract('a1', '(\d+)', NULL); +NULL +```"#, + argument( + name = "str", + description = "String expression to search. Values of other types, such as numbers and binary, are cast to a string. NULL returns NULL." + ), + argument( + name = "regexp", + description = "String regular expression in Rust `regex` syntax. Values of other types are cast to a string. NULL returns NULL." + ), + argument( + name = "idx", + description = "Optional capture group index, cast to INT (numeric and string values are accepted). Defaults to 1; 0 is the whole match. NULL returns NULL." + ) +)] +#[derive(Debug)] +pub struct SparkRegexpExtract { + signature: Signature, + /// Literal patterns compiled by earlier batches. One instance serves every + /// query in a session, so the cache is keyed by pattern. + patterns: PatternCache, +} + +// The cache does not affect the result, so two instances with the same +// signature are equal. +impl PartialEq for SparkRegexpExtract { + fn eq(&self, other: &Self) -> bool { + self.signature == other.signature + } +} + +impl Eq for SparkRegexpExtract {} + +impl Hash for SparkRegexpExtract { + fn hash(&self, state: &mut H) { + self.signature.hash(state); + } +} + +/// The literal patterns kept compiled. When it is full it is cleared, which +/// bounds its size if many different literal patterns are used. +const PATTERN_CACHE_SIZE: usize = 64; + +#[derive(Debug, Default)] +struct PatternCache { + patterns: Mutex>, + /// How many patterns were compiled, so a test can tell that a later batch + /// read the cache rather than compiling and inserting the pattern again. + #[cfg(test)] + compiles: std::sync::atomic::AtomicUsize, +} + +impl PatternCache { + fn get_or_compile(&self, pattern: &str) -> Result { + let lock = || { + self.patterns + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + }; + if let Some(regex) = lock().get(pattern) { + return Ok(regex.clone()); + } + #[cfg(test)] + self.compiles + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let regex = compile(pattern)?; + let mut patterns = lock(); + if patterns.len() >= PATTERN_CACHE_SIZE { + patterns.clear(); + } + patterns.insert(pattern.to_string(), regex.clone()); + Ok(regex) + } +} + +impl Default for SparkRegexpExtract { + fn default() -> Self { + Self::new() + } +} + +impl SparkRegexpExtract { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + patterns: PatternCache::default(), + } + } +} + +impl ScalarUDFImpl for SparkRegexpExtract { + fn name(&self) -> &str { + NAME + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Utf8) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + if !(2..=3).contains(&arg_types.len()) { + return Err(invalid_arg_count_exec_err(NAME, (2, 3), arg_types.len())); + } + // Like Spark's implicit casts: str and regexp are cast to strings, idx to INT. + let castable_to_string = |t: &DataType| { + *t == DataType::Null || (!t.is_nested() && can_cast_types(t, &DataType::Utf8)) + }; + for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) { + if !castable_to_string(arg_type) { + return plan_err!("{NAME} expects a string {name}, got {arg_type}"); + } + } + let mut coerced = vec![DataType::Utf8, DataType::Utf8]; + if let Some(idx) = arg_types.get(2) { + let castable_to_int = idx.is_numeric() + || matches!( + idx, + DataType::Null + | DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View + ); + if !castable_to_int { + return plan_err!("{NAME} expects an INT idx, got {idx}"); + } + coerced.push(DataType::Int32); + } + Ok(coerced) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let all_scalar = args + .args + .iter() + .all(|arg| matches!(arg, ColumnarValue::Scalar(_))); + let len = if all_scalar { 1 } else { args.number_rows }; + + // The usual call has a literal pattern and idx: keep those as scalars so + // the pattern is compiled once and no per-row copies are allocated. + let pattern_column = match &args.args[1] { + ColumnarValue::Scalar(_) => None, + ColumnarValue::Array(array) => Some(to_utf8("regexp", array)?), + }; + let pattern = match (&args.args[1], &pattern_column) { + (_, Some(patterns)) => Pattern::Array(patterns.as_string::()), + (ColumnarValue::Scalar(scalar), None) => match scalar_str(scalar)? { + Some(pattern) => Pattern::Scalar(pattern), + None => return null_result(len, all_scalar), + }, + (ColumnarValue::Array(_), None) => unreachable!("converted above"), + }; + let index = match args.args.get(2) { + None => Index::Scalar(1), + Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => { + Index::Scalar(*idx) + } + Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | ScalarValue::Null)) => { + return null_result(len, all_scalar); + } + Some(ColumnarValue::Scalar(other)) => { + return internal_err!( + "{NAME} expects an INT idx, got {}", + other.data_type() + ); + } + Some(ColumnarValue::Array(array)) => Index::Array(as_int32_array(array)?), + }; + + // A caller that builds the physical expression directly, such as Comet, + // can pass any string type. Each result is a substring of its input, so + // the bytes of the visible input values bound the output. + let strings = args.args[0].to_array(len)?; + let result = match strings.data_type() { + DataType::Utf8 => { + let strings = strings.as_string::(); + let capacity = offset_span_len(strings.offsets()); + extract(strings, capacity, pattern, index, &self.patterns)? + } + DataType::LargeUtf8 => { + let strings = strings.as_string::(); + let capacity = offset_span_len(strings.offsets()); + extract(strings, capacity, pattern, index, &self.patterns)? + } + DataType::Utf8View => { + let strings = strings.as_string_view(); + let capacity = strings.lengths().map(|len| len as usize).sum(); + extract(strings, capacity, pattern, index, &self.patterns)? + } + other => return exec_err!("{NAME} expects a string str, got {other}"), + }; + if all_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } +} + +#[derive(Clone, Copy)] +enum Pattern<'a> { + Scalar(&'a str), + Array(&'a StringArray), +} + +#[derive(Clone, Copy)] +enum Index<'a> { + Scalar(i32), + Array(&'a Int32Array), +} + +/// A pattern column as `Utf8`. Other string types are cast, and anything else +/// is an error. +fn to_utf8(name: &str, array: &ArrayRef) -> Result { + match array.data_type() { + DataType::Utf8 => Ok(Arc::clone(array)), + DataType::LargeUtf8 | DataType::Utf8View => Ok(cast(array, &DataType::Utf8)?), + other => exec_err!("{NAME} expects a string {name}, got {other}"), + } +} + +fn scalar_str(scalar: &ScalarValue) -> Result> { + match scalar { + ScalarValue::Null => Ok(None), + other => other.try_as_str().ok_or_else(|| { + internal_datafusion_err!( + "{NAME} expects a string regexp, got {}", + other.data_type() + ) + }), + } +} + +fn null_result(len: usize, all_scalar: bool) -> Result { + Ok(if all_scalar { + ColumnarValue::Scalar(ScalarValue::Utf8(None)) + } else { + ColumnarValue::Array(new_null_array(&DataType::Utf8, len)) + }) +} + +fn extract<'a, S: StringArrayType<'a> + Copy>( + strings: S, + capacity: usize, + pattern: Pattern, + index: Index, + cache: &PatternCache, +) -> Result { + let mut builder = StringBuilder::with_capacity(strings.len(), capacity); + // A literal pattern comes from the cache on the first non-NULL row, so a NULL + // row is not checked. A column pattern keeps only the last one compiled, + // since it rarely changes per row. The capture buffer is reused across rows + // instead of allocated per match. + let mut cached: Option<(&str, Regex, CaptureLocations)> = None; + + for row in 0..strings.len() { + let row_pattern = match pattern { + Pattern::Scalar(pattern) => Some(pattern), + Pattern::Array(patterns) => { + (!patterns.is_null(row)).then(|| patterns.value(row)) + } + }; + let idx = match index { + Index::Scalar(idx) => Some(idx), + Index::Array(indexes) => (!indexes.is_null(row)).then(|| indexes.value(row)), + }; + let (false, Some(pattern_str), Some(idx)) = + (strings.is_null(row), row_pattern, idx) + else { + builder.append_null(); + continue; + }; + let (_, regex, locations) = match &mut cached { + Some(entry) if entry.0 == pattern_str => entry, + _ => { + let regex = match pattern { + Pattern::Scalar(_) => cache.get_or_compile(pattern_str)?, + Pattern::Array(_) => compile(pattern_str)?, + }; + let locations = regex.capture_locations(); + cached.insert((pattern_str, regex, locations)) + } + }; + let subject = strings.value(row); + // Like Spark, idx is only checked when the pattern matches. + let text = match regex.captures_read(locations, subject) { + None => "", + Some(_) => locations + .get(check_group_index(regex, idx)?) + .map_or("", |(start, end)| &subject[start..end]), + }; + builder.append_value(text); + } + Ok(Arc::new(builder.finish())) +} + +fn check_group_index(regex: &Regex, idx: i32) -> Result { + let group_count = regex.captures_len() - 1; + usize::try_from(idx) + .ok() + .filter(|group| *group <= group_count) + .ok_or_else(|| { + exec_datafusion_err!( + "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of parameter(s) \ + `idx` in `{NAME}` is invalid: Expects group index between 0 and \ + {group_count}, but got {idx}. SQLSTATE: 22023" + ) + }) +} + +fn compile(pattern: &str) -> Result { + let quoted = to_sql_value(pattern); + let unsupported = |reason: String| { + DataFusionError::NotImplemented(format!( + "Unsupported regex pattern {quoted} in `{NAME}`: {reason}. \ + See https://github.com/apache/datafusion-spark/issues/255" + )) + }; + match java_difference(pattern) { + None => {} + Some(JavaDifference::Invalid) => { + return Err(exec_datafusion_err!( + "[INVALID_PARAMETER_VALUE.PATTERN] The value of parameter(s) `regexp` in \ + `{NAME}` is invalid: {quoted}. SQLSTATE: 22023" + )); + } + Some(JavaDifference::Unsupported(reason)) => return Err(unsupported(reason)), + } + // The pattern parsed, so a later stage of the Rust engine rejected it. + Regex::new(pattern).map_err(|err| match err { + regex::Error::CompiledTooBig(_) => unsupported( + "the compiled pattern exceeds the Rust regex size limit".to_string(), + ), + err => unsupported(format!("the Rust regex engine rejects it ({err})")), + }) +} + +/// How `java.util.regex`, which Spark uses, would treat a pattern differently +/// from the Rust regex engine. +#[derive(Debug)] +enum JavaDifference { + /// Java rejects the pattern, so it is reported as an invalid pattern. + Invalid, + /// Java accepts the pattern but the Rust engine rejects it, or reads it + /// differently, so it is reported as unsupported rather than evaluated to + /// a different answer than Spark's. + Unsupported(String), +} + +/// Checks a pattern with `regex_syntax`, whose AST and error kinds are API, +/// rather than matching on the text of [`regex::Error`]. +/// +/// A pattern that does not parse is invalid only for the kinds of syntax error +/// that Java also raises. Anything else, such as `\Q...\E`, `\Z`, `\h`, +/// `(?>...)` or `(?d)`, is unsupported, which also covers a few patterns that +/// Java rejects, such as `\y`. A pattern that parses is walked for the +/// constructs that the two engines read differently. +fn java_difference(pattern: &str) -> Option { + use regex_syntax::ast::ErrorKind; + let ast_err = match regex_syntax::ast::parse::Parser::new().parse(pattern) { + Ok(ast) => return regex_syntax::ast::visit(&ast, JavaDifferences).err(), + Err(err) => err, + }; + let unsupported = |reason: String| Some(JavaDifference::Unsupported(reason)); + match ast_err.kind() { + ErrorKind::GroupUnclosed + | ErrorKind::GroupUnopened + | ErrorKind::ClassUnclosed + | ErrorKind::ClassRangeInvalid + | ErrorKind::RepetitionMissing + | ErrorKind::RepetitionCountInvalid => Some(JavaDifference::Invalid), + // Java also rejects an unknown flag, except `d` (UNIX_LINES) and `>` + // (an atomic group). + ErrorKind::FlagUnrecognized => { + let span = ast_err.span(); + let flag = &pattern[span.start.offset..span.end.offset]; + if matches!(flag, "d" | ">") { + unsupported(format!( + "the Rust regex engine does not support the flag '{flag}'" + )) + } else { + Some(JavaDifference::Invalid) + } + } + ErrorKind::UnsupportedLookAround => unsupported( + "look-around is not supported by the Rust regex engine".to_string(), + ), + ErrorKind::UnsupportedBackreference => unsupported( + "backreferences are not supported by the Rust regex engine".to_string(), + ), + kind => unsupported(format!( + "the Rust regex engine does not support it ({kind})" + )), + } +} + +/// Java's POSIX character classes, which match ASCII only (unless the `U` flag +/// is set, which is unsupported). The Rust engine reads the same names as +/// Unicode properties, so `\p{Alpha}` matches `é` there but not in Java. +const JAVA_POSIX_CLASSES: [&str; 13] = [ + "Lower", "Upper", "ASCII", "Alpha", "Digit", "Alnum", "Punct", "Graph", "Print", + "Blank", "Cntrl", "XDigit", "Space", +]; + +/// Finds the constructs of a parsed pattern that Java rejects or reads +/// differently. Each is checked by its AST node, so an escaped or quoted +/// look-alike, such as `\[\[:alpha:\]\]`, is not reported. +struct JavaDifferences; + +impl JavaDifferences { + fn check_flags(flags: &Flags) -> Result<(), JavaDifference> { + // Java has no R flag, even negated. + if flags + .items + .iter() + .any(|item| item.kind == FlagsItemKind::Flag(Flag::CRLF)) + { + return Err(JavaDifference::Invalid); + } + // `(?-U)` is the default in both engines, so only setting it differs. + if flags.flag_state(Flag::SwapGreed) == Some(true) { + return Err(JavaDifference::Unsupported( + "Java's flag 'U' is UNICODE_CHARACTER_CLASS, where the Rust regex \ + engine's swaps greediness" + .to_string(), + )); + } + Ok(()) + } + + fn check_unicode_class(class: &ClassUnicode) -> Result<(), JavaDifference> { + match &class.kind { + ClassUnicodeKind::Named(name) + if JAVA_POSIX_CLASSES.contains(&name.as_str()) => + { + let p = if class.negated { 'P' } else { 'p' }; + Err(JavaDifference::Unsupported(format!( + "Java's \\{p}{{{name}}} is an ASCII-only POSIX class, where the Rust \ + regex engine's is a Unicode property" + ))) + } + _ => Ok(()), + } + } +} + +impl Visitor for JavaDifferences { + type Output = (); + type Err = JavaDifference; + + fn finish(self) -> Result<(), JavaDifference> { + Ok(()) + } + + fn visit_pre(&mut self, ast: &Ast) -> Result<(), JavaDifference> { + match ast { + Ast::Flags(set) => Self::check_flags(&set.flags), + Ast::Group(group) => match &group.kind { + GroupKind::NonCapturing(flags) => Self::check_flags(flags), + // Java rejects `(?P...)`, and a name that is not an ASCII + // letter followed by ASCII letters and digits. + GroupKind::CaptureName { starts_with_p, name } => { + let mut chars = name.name.chars(); + let valid = !starts_with_p + && chars.next().is_some_and(|c| c.is_ascii_alphabetic()) + && chars.all(|c| c.is_ascii_alphanumeric()); + if valid { + Ok(()) + } else { + Err(JavaDifference::Invalid) + } + } + GroupKind::CaptureIndex(_) => Ok(()), + }, + Ast::Assertion(assertion) => match assertion.kind { + // Java rejects `\b{start}`, `\b{end}`, `\b{start-half}` and + // `\b{end-half}`. + AssertionKind::WordBoundaryStart + | AssertionKind::WordBoundaryEnd + | AssertionKind::WordBoundaryStartHalf + | AssertionKind::WordBoundaryEndHalf => Err(JavaDifference::Invalid), + AssertionKind::WordBoundaryStartAngle + | AssertionKind::WordBoundaryEndAngle => Err(JavaDifference::Unsupported( + "Java reads \\< and \\> as literal characters, where the Rust regex \ + engine reads them as word boundaries" + .to_string(), + )), + AssertionKind::StartLine + | AssertionKind::EndLine + | AssertionKind::StartText + | AssertionKind::EndText + | AssertionKind::WordBoundary + | AssertionKind::NotWordBoundary => Ok(()), + }, + Ast::ClassUnicode(class) => Self::check_unicode_class(class), + _ => Ok(()), + } + } + + fn visit_class_set_item_pre( + &mut self, + item: &ClassSetItem, + ) -> Result<(), JavaDifference> { + match item { + ClassSetItem::Ascii(_) => Err(JavaDifference::Unsupported( + "Java reads [:name:] inside a class as a nested class of its characters, \ + where the Rust regex engine reads it as a POSIX class" + .to_string(), + )), + ClassSetItem::Unicode(class) => Self::check_unicode_class(class), + _ => Ok(()), + } + } + + fn visit_class_set_binary_op_pre( + &mut self, + op: &ClassSetBinaryOp, + ) -> Result<(), JavaDifference> { + let operator = match op.kind { + // `&&` is intersection in both engines. + ClassSetBinaryOpKind::Intersection => return Ok(()), + ClassSetBinaryOpKind::Difference => "--", + ClassSetBinaryOpKind::SymmetricDifference => "~~", + }; + Err(JavaDifference::Unsupported(format!( + "Java has no class operator '{operator}', which it reads as literal characters" + ))) + } +} + +/// Quotes a string the way Spark's `toSQLValue` renders a string literal in an +/// error message: wrapped in single quotes, with `\` and `'` escaped. +fn to_sql_value(value: &str) -> String { + format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{LargeStringArray, StringViewArray}; + use arrow::datatypes::Field; + use datafusion_common::config::ConfigOptions; + + fn invoke( + udf: &SparkRegexpExtract, + args: Vec, + number_rows: usize, + ) -> Result { + let arg_fields = args + .iter() + .map(|arg| Arc::new(Field::new("a", arg.data_type(), true))) + .collect(); + udf.invoke_with_args(ScalarFunctionArgs { + args, + arg_fields, + number_rows, + return_field: Arc::new(Field::new(NAME, DataType::Utf8, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn pattern(pattern: &str) -> ColumnarValue { + ColumnarValue::Scalar(ScalarValue::Utf8(Some(pattern.to_string()))) + } + + fn strings(result: ColumnarValue) -> Vec> { + let array = result.into_array(1).unwrap(); + array + .as_string::() + .iter() + .map(|s| s.map(str::to_string)) + .collect() + } + + /// Ported from Comet's `large_utf8_subject_returns_utf8_array`: a caller + /// that builds the physical expression directly can pass any string type, + /// and the result is always `Utf8`. + #[test] + fn large_utf8_and_view_subjects_return_utf8() { + let udf = SparkRegexpExtract::new(); + let subjects: [ArrayRef; 2] = [ + Arc::new(LargeStringArray::from(vec![Some("a1"), None, Some("b22")])), + Arc::new(StringViewArray::from(vec![Some("a1"), None, Some("b22")])), + ]; + for subject in subjects { + let result = invoke( + &udf, + vec![ColumnarValue::Array(subject), pattern(r"(\d+)")], + 3, + ) + .unwrap(); + assert_eq!(result.data_type(), DataType::Utf8); + assert_eq!( + strings(result), + vec![Some("1".to_string()), None, Some("22".to_string())] + ); + } + } + + #[test] + fn large_utf8_pattern_column() { + let udf = SparkRegexpExtract::new(); + let subject: ArrayRef = Arc::new(StringArray::from(vec!["a1", "b2"])); + let patterns: ArrayRef = + Arc::new(LargeStringArray::from(vec![r"(\d)", "([a-z])"])); + let result = invoke( + &udf, + vec![ + ColumnarValue::Array(subject), + ColumnarValue::Array(patterns), + ], + 2, + ) + .unwrap(); + assert_eq!( + strings(result), + vec![Some("1".to_string()), Some("b".to_string())] + ); + } + + #[test] + fn non_string_subject_is_an_error() { + let udf = SparkRegexpExtract::new(); + let subject: ArrayRef = Arc::new(Int32Array::from(vec![1])); + let error = invoke(&udf, vec![ColumnarValue::Array(subject), pattern("(1)")], 1) + .unwrap_err(); + assert!( + error + .to_string() + .contains("expects a string str, got Int32"), + "{error}" + ); + } + + /// The output is sized from the visible values, not the whole buffer that + /// a slice shares with its parent. + #[test] + fn sliced_subject_allocates_for_the_slice() { + let udf = SparkRegexpExtract::new(); + let padding = "x".repeat(65536); + let parent = StringArray::from(vec![padding.as_str(), "a1", padding.as_str()]); + for (offset, len, expected) in + [(1, 0, vec![]), (1, 1, vec![Some("1".to_string())])] + { + let subject: ArrayRef = Arc::new(parent.slice(offset, len)); + let result = invoke( + &udf, + vec![ColumnarValue::Array(subject), pattern(r"(\d)")], + len, + ) + .unwrap() + .into_array(len) + .unwrap(); + let result = result.as_string::(); + assert_eq!( + result + .iter() + .map(|s| s.map(str::to_string)) + .collect::>(), + expected + ); + assert!( + result.values().capacity() < 1024, + "allocated {} bytes", + result.values().capacity() + ); + } + } + + #[test] + fn literal_patterns_are_cached_across_batches() { + let udf = SparkRegexpExtract::new(); + for subject in ["a1", "b2"] { + let subject = + ColumnarValue::Array(Arc::new(StringArray::from(vec![subject]))); + invoke(&udf, vec![subject, pattern(r"(\d)")], 1).unwrap(); + } + // Counting compiles, not cache entries, since a cache that is never + // read would also end up with one entry for the pattern. + assert_eq!( + udf.patterns + .compiles + .load(std::sync::atomic::Ordering::Relaxed), + 1 + ); + assert!(udf.patterns.patterns.lock().unwrap().contains_key(r"(\d)")); + } +} diff --git a/crates/sqllogictest/test_files/string/regexp_extract.slt b/crates/sqllogictest/test_files/string/regexp_extract.slt new file mode 100644 index 00000000..7e618d33 --- /dev/null +++ b/crates/sqllogictest/test_files/string/regexp_extract.slt @@ -0,0 +1,567 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible regexp_extract +# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract +# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out +# or were verified against Spark 4.2.0. +# Audited against Spark 4.2.0 on 2026-11-15. +# Expected values verified with pyspark==4.2.0. +# Known divergences are captured as commented-out queries below. +# Patterns use the Rust regex engine, not java.util.regex; the differences are +# tracked by https://github.com/apache/datafusion-spark/issues/255 +# The REGEX_GROUP_INDEX error matches Spark 3.5 and later (Spark 3.4 raised +# _LEGACY_ERROR_TEMP_2006 / _LEGACY_ERROR_TEMP_2007 instead). + +query T +SELECT regexp_extract('1a 2b 14m', '\d+', 0); +---- +1 + +# Default group index is 1 +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)'); +---- +1 + +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0); +---- +1a + +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 1); +---- +1 + +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2); +---- +a + +# A group that did not take part in the match, or no match at all, gives '' +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)?', 1); +---- +1 + +query T +SELECT regexp_extract('a b m', '(\d+)?', 1); +---- +(empty) + +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)?([a-z]+)', 1); +---- +1 + +query T +SELECT regexp_extract('a b m', '(\d+)?([a-z]+)', 1); +---- +(empty) + +query T +SELECT regexp_extract('abc', '(\d+)', 1); +---- +(empty) + +# idx is only checked when the pattern matches, so no match gives '' even +# when idx is out of range (including the default idx 1 with no groups) +query T +SELECT regexp_extract('abc', '\d+'); +---- +(empty) + +query T +SELECT regexp_extract('abc', '\d+', 5); +---- +(empty) + +query T +SELECT regexp_extract('abc', '\d+', -1); +---- +(empty) + +query T +SELECT regexp_extract('abc', '(\d+)', 2); +---- +(empty) + +query T +SELECT regexp_extract('', '\d+', 3); +---- +(empty) + +# NULL in any argument gives NULL +query T +SELECT regexp_extract(NULL, '(\d+)', 1); +---- +NULL + +query T +SELECT regexp_extract('a1', NULL, 1); +---- +NULL + +query T +SELECT regexp_extract('a1', '(\d+)', NULL); +---- +NULL + +# A NULL row returns NULL before idx or the pattern is checked +query T +SELECT regexp_extract(NULL, '(unclosed', 0); +---- +NULL + +query T +SELECT regexp_extract('a', '(unclosed', NULL); +---- +NULL + +# A NULL idx returns NULL before idx is checked +query T +SELECT regexp_extract('a', NULL, 99); +---- +NULL + +# Group index out of range +statement error ^.*\[INVALID_PARAMETER_VALUE\.REGEX_GROUP_INDEX\] The value of parameter\(s\) `idx` in `regexp_extract` is invalid: Expects group index between 0 and 0, but got 1\. SQLSTATE: 22023$ +SELECT regexp_extract('1a 2b 14m', '\d+'); + +statement error Expects group index between 0 and 0, but got 1 +SELECT regexp_extract('1a 2b 14m', '\d+', 1); + +statement error Expects group index between 0 and 0, but got 2 +SELECT regexp_extract('1a 2b 14m', '\d+', 2); + +statement error Expects group index between 0 and 0, but got -1 +SELECT regexp_extract('1a 2b 14m', '\d+', -1); + +statement error Expects group index between 0 and 2, but got 3 +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 3); + +statement error Expects group index between 0 and 2, but got -1 +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1); + +# Invalid pattern +statement error ^.*\[INVALID_PARAMETER_VALUE\.PATTERN\] The value of parameter\(s\) `regexp` in `regexp_extract` is invalid: '\(\?l\)'\. SQLSTATE: 22023$ +SELECT regexp_extract('1a 2b 14m', '(?l)'); + +statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed' +SELECT regexp_extract('x', '(unclosed', 0); + +# The pattern is quoted like Spark's toSQLValue, with \ and ' escaped +statement error is invalid: '\(\\\\d'\. +SELECT regexp_extract('x', '(\d', 0); + +statement error is invalid: 'it\\'s\('\. +SELECT regexp_extract('x', 'it''s(', 0); + +# An empty pattern matches the empty string +query T +SELECT regexp_extract('abc', '', 0); +---- +(empty) + +# A zero-width group, and a group that ends after a multi-byte character +query TT +SELECT regexp_extract('éx', '()', 1), regexp_extract('éx', '(é)', 1); +---- +(empty) é + +# Known differences between the Rust regex engine and java.util.regex +# https://github.com/apache/datafusion-spark/issues/255 + +# Look-around and backreferences are not supported by the Rust regex engine +statement error Unsupported regex pattern 'a\(\?=b\)' in `regexp_extract`: look-around +SELECT regexp_extract('ab', 'a(?=b)', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: backreferences +SELECT regexp_extract('aa', '(a)\1', 0); + +# Spark 4.2.0 supports look-around +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('ab', 'a(?=b)', 0); +# ---- +# a + +# Spark 4.2.0 supports backreferences +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('aa', '(a)\1', 0); +# ---- +# aa + +# Java constructs that the Rust regex engine does not have are unsupported, not +# invalid: quoting, \Z, \h, atomic groups and the d flag +statement error Unsupported regex pattern '\\\\Q\.\\\\E' in `regexp_extract` +SELECT regexp_extract('a.b', '\Q.\E', 0); + +statement error Unsupported regex pattern .* in `regexp_extract` +SELECT regexp_extract('ab', 'b\Z', 0); + +statement error Unsupported regex pattern .* in `regexp_extract` +SELECT regexp_extract('a b', '\h', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: the Rust regex engine does not support the flag '>' +SELECT regexp_extract('aab', '(?>a+)b', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: the Rust regex engine does not support the flag 'd' +SELECT regexp_extract('ab', '(?d)b', 0); + +# Spark 4.2.0 results for those patterns, in both ANSI modes +# https://github.com/apache/datafusion-spark/issues/255 +# query TTTTT +# SELECT regexp_extract('a.b', '\Q.\E', 0), regexp_extract('ab', 'b\Z', 0), concat('[', regexp_extract('a b', '\h', 0), ']'), regexp_extract('aab', '(?>a+)b', 0), regexp_extract('ab', '(?d)b', 0); +# ---- +# . b [ ] aab b + +# Spark 4.2.0 treats a++ as a possessive quantifier that does not give back, +# so a++a cannot match; the Rust regex engine reads a++ as (a+)+ and returns 'aaa' +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('aaa', '(a++a)', 1); +# ---- +# (empty) + +# A pattern over the Rust regex engine's compiled-size limit is not supported +statement error Unsupported regex pattern .* in `regexp_extract`: the compiled pattern exceeds the Rust regex size limit +SELECT regexp_extract('a', '\w{1000}\w{1000}\w{1000}', 0); + +# Spark 4.2.0 matches \d against ASCII digits only; DataFusion returns '٣' +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('x٣y', '\d', 0); +# ---- +# (empty) + +# Spark 4.2.0's . does not match a carriage return; DataFusion returns 'a\rb' +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('a' || chr(13) || 'b', '(a.b)', 1); +# ---- +# (empty) + +# Spark 4.2.0's $ also matches before a final line terminator; DataFusion returns '' +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('abc' || chr(10), '(abc$)', 1); +# ---- +# abc + +# Spark 4.2.0's (?i) folds ASCII case only, so k does not match the Kelvin sign +# (U+212A); DataFusion returns it +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('K', '((?i)k)', 1); +# ---- +# (empty) + +# Patterns that the Rust regex engine accepts but Java rejects are invalid, as +# in Spark 4.2.0: the R flag, \b{...} word boundaries, Python-style named groups +# and group names that are not an ASCII letter followed by letters and digits +statement error ^.*\[INVALID_PARAMETER_VALUE\.PATTERN\] The value of parameter\(s\) `regexp` in `regexp_extract` is invalid: '\(\?R\)a\$'\. SQLSTATE: 22023$ +SELECT regexp_extract('a', '(?R)a$', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('a', '(?-R)a', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('abc', '\b{start}abc', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('abc', 'abc\b{end}', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('abc', '\b{start-half}abc', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('abc', 'abc\b{end-half}', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN.*is invalid: '\(\?Pa\)'\. +SELECT regexp_extract('a', '(?Pa)', 1); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('x', '(?x)', 1); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('x', '(?<_a>x)', 1); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('x', '(?x)', 1); + +query T +SELECT regexp_extract('x', '(?x)', 1); +---- +x + +# Patterns that both engines accept but read differently are unsupported, so +# they do not silently return a different answer than Spark. Java's U flag is +# UNICODE_CHARACTER_CLASS, where the Rust engine's swaps greediness. +statement error Unsupported regex pattern '\(\?U\)a\+' in `regexp_extract`: Java's flag 'U' +SELECT regexp_extract('aaa', '(?U)a+', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java's flag 'U' +SELECT regexp_extract('aaa', '(?U:a+)', 0); + +# Java reads [[:alpha:]] as a nested class of the characters ':alph' +statement error Unsupported regex pattern .* in `regexp_extract`: Java reads \[:name:\] +SELECT regexp_extract('x', '[[:alpha:]]', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java reads \[:name:\] +SELECT regexp_extract('ab', '[[:^alpha:]]', 0); + +# Java has no class difference operators +statement error Unsupported regex pattern .* in `regexp_extract`: Java has no class operator '--' +SELECT regexp_extract('b', '[a-c--b]', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java has no class operator '~~' +SELECT regexp_extract('~', '[a~~b]', 0); + +# Java reads \< and \> as the literal characters +statement error Unsupported regex pattern .* in `regexp_extract`: Java reads \\< and \\> as literal characters +SELECT regexp_extract('a as literal characters +SELECT regexp_extract('a>', 'a\>', 0); + +# Java's POSIX classes match ASCII only, where the Rust engine reads the same +# names as Unicode properties +statement error Unsupported regex pattern '\\\\p\{Alpha\}' in `regexp_extract`: Java's \\p\{Alpha\} is an ASCII-only POSIX class +SELECT regexp_extract('é', '\p{Alpha}', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java's \\P\{Lower\} is an ASCII-only POSIX class +SELECT regexp_extract('é', '\P{Lower}', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java's \\p\{Space\} is an ASCII-only POSIX class +SELECT regexp_extract('a', '[a\p{Space}]', 0); + +# A pattern column is checked the same way +statement error Unsupported regex pattern .* in `regexp_extract`: Java's flag 'U' +SELECT regexp_extract(column1, column2, 0) FROM (VALUES ('aaa', 'a+'), ('aaa', '(?U)a+')); + +# Spark 4.2.0 results for those patterns, in both ANSI modes +# https://github.com/apache/datafusion-spark/issues/255 +# query TTTTTTTTTTT +# SELECT regexp_extract('aaa', '(?U)a+', 0), concat('[', regexp_extract('x', '[[:alpha:]]', 0), ']'), regexp_extract(':', '[[:alpha:]]', 0), regexp_extract('ab', '[[:^alpha:]]', 0), regexp_extract('b', '[a-c--b]', 0), regexp_extract('~', '[a~~b]', 0), regexp_extract('a', 'a\>', 0), concat('[', regexp_extract('é', '\p{Alpha}', 0), ']'), regexp_extract('é', '\P{Lower}', 0), regexp_extract('1', '\p{Digit}', 0); +# ---- +# aaa [] : a b ~ [] é 1 + +# Constructs that mean the same in both engines are accepted: the default (?-U), +# class intersection and Unicode properties that are not POSIX names +query TTT +SELECT regexp_extract('aaa', '(?-U)a+', 0), regexp_extract('b', '[a-c&&b]', 0), regexp_extract('é', '\p{L}', 0); +---- +aaa b é + +# Spark 4.2.0 rejects Unicode property names that Java does not know, such as +# \p{Letter}, \p{Latin} and \p{alpha}; DataFusion returns 'é' +# https://github.com/apache/datafusion-spark/issues/255 +# statement error INVALID_PARAMETER_VALUE.PATTERN +# SELECT regexp_extract('é', '\p{Letter}', 0); + +# Wrong number of arguments +statement error +SELECT regexp_extract('abc'); + +# Column input, including NULL rows and a per-row pattern +statement ok +CREATE TABLE regexp_extract_t AS VALUES ('100-200', '(\d+)-(\d+)'), (NULL, '(\d+)'), ('abc', '([a-z])'); + +query T +SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_t; +---- +100 +NULL +a + +# Per-row patterns alternating between rows, with NULL string and pattern rows +statement ok +CREATE TABLE regexp_extract_patterns AS VALUES ('a1', '(\d)'), (NULL, '(\d)'), ('b2', NULL), ('c3', '([a-z])'), ('d4', '(\d)'), ('e5', '([a-z])(\d)'); + +query T +SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_patterns; +---- +1 +NULL +NULL +c +4 +e + +# Per-row idx, including NULL and an out-of-range idx on a row that does not match +statement ok +CREATE TABLE regexp_extract_idx AS VALUES ('a1', '(\d)', 1), ('d4', '(\d)', NULL), ('xy', '(\d)', 7), ('c3', '([a-z])(\d)', 2); + +query T +SELECT regexp_extract(column1, column2, column3) FROM regexp_extract_idx; +---- +1 +NULL +(empty) +3 + +query T +SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), ('def', -1), (NULL, 9)); +---- +(empty) +(empty) +NULL + +# An out-of-range idx fails once a row matches +statement error Expects group index between 0 and 1, but got 3 +SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), ('12', 3)); + +# Literal str with a column pattern +query T +SELECT regexp_extract('a1', column1, 1) FROM (VALUES ('(\d)'), ('([a-z])')); +---- +1 +a + +# Literal str and pattern with a column idx +query T +SELECT regexp_extract('a1b2', '([a-z])(\d)', column1) FROM (VALUES (0), (1), (2)); +---- +a1 +a +1 + +# An invalid pattern in a later row of a pattern column is an error +statement error INVALID_PARAMETER_VALUE.PATTERN.*is invalid: '\('\. +SELECT regexp_extract(column1, column2, 0) FROM (VALUES ('a', 'a'), ('b', '(')); + +# A NULL str row is not checked against its invalid pattern +query T +SELECT regexp_extract(column1, column2, 0) FROM (VALUES (CAST(NULL AS STRING), '('), ('b', 'b')); +---- +NULL +b + +# Literal pattern with no match on any row and the default idx +query T +SELECT regexp_extract(column1, '\d+') FROM (VALUES ('abc'), ('xyz')); +---- +(empty) +(empty) + +# An invalid literal pattern is not compiled when every string is NULL, as in +# Spark, so a literal pattern is not validated at planning time +query T +SELECT regexp_extract(column1, '(unclosed', 0) FROM (VALUES (CAST(NULL AS STRING)), (CAST(NULL AS STRING))); +---- +NULL +NULL + +# Other string types are accepted +query T +SELECT regexp_extract(arrow_cast('1a 2b', 'LargeUtf8'), arrow_cast('(\d+)([a-z])', 'Utf8View'), 2); +---- +a + +# Like Spark, str and regexp of other types are cast to strings +query T +SELECT regexp_extract(12345, '(\d)', 1); +---- +1 + +query T +SELECT regexp_extract(column1, '(\d)', 1) FROM (VALUES (12345), (67)); +---- +1 +6 + +query T +SELECT regexp_extract(X'6131', '(\d)', 1); +---- +1 + +query T +SELECT regexp_extract('a1', 1, 0); +---- +1 + +# Like Spark, idx of other numeric types and strings is cast to INT +query T +SELECT regexp_extract('1a 2b', '(\d+)([a-z])', arrow_cast(2, 'Int8')); +---- +a + +query T +SELECT regexp_extract('a1', '(\d)', '1'); +---- +1 + +query T +SELECT regexp_extract('a1', '(\d)', 1.0::DOUBLE); +---- +1 + +query T +SELECT regexp_extract('a1', '(\d)', 1.5::DOUBLE); +---- +1 + +# A string idx column is cast to INT +query T +SELECT regexp_extract('a1', '([a-z])(\d)', column1) FROM (VALUES ('0'), ('1'), ('2'), (CAST(NULL AS STRING))); +---- +a1 +a +1 +NULL + +# With ANSI mode on, an idx that cannot be cast to INT is an error, as in Spark +# (Spark raises CAST_OVERFLOW and CAST_INVALID_INPUT) +statement ok +set datafusion.execution.enable_ansi_mode = true; + +statement error Can't cast value 4294967297 to type Int32 +SELECT regexp_extract('a1', '(\d)', 4294967297::BIGINT); + +statement error Cannot cast string 'x' to value of Int32 type +SELECT regexp_extract('a1', '(\d)', 'x'); + +statement ok +set datafusion.execution.enable_ansi_mode = false; + +# With ANSI mode off, Spark 4.2.0 wraps an out-of-range BIGINT (to 1 here), returns +# NULL for a malformed string, and casts NaN to 0. DataFusion's cast errors instead. +# https://github.com/apache/datafusion-spark/issues/88 +# query T +# SELECT regexp_extract('a1', '(\d)', 4294967297::BIGINT); +# ---- +# 1 +# +# query T +# SELECT regexp_extract('a1', '(\d)', 'x'); +# ---- +# NULL +# +# query T +# SELECT regexp_extract('a1', '(\d)', 'NaN'::DOUBLE); +# ---- +# 1 + +# A str that cannot be cast to a string is a planning error, as in Spark +# (Spark raises DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE) +statement error Error during planning: regexp_extract expects a string str, got List +SELECT regexp_extract(array[1], 'a', 0); + +# The result type is a string +query T +SELECT arrow_typeof(regexp_extract('a', '(a)')); +---- +Utf8 + +statement ok +DROP TABLE regexp_extract_t; + +statement ok +DROP TABLE regexp_extract_patterns; + +statement ok +DROP TABLE regexp_extract_idx;