From 2bdf9f6a0b9afec4937e2196b788ff6a82d80465 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 09:15:31 -0700 Subject: [PATCH 1/8] feat(spark): add Spark-compatible regexp_extract function Port Comet's regexp_extract. --- Cargo.lock | 1 + Cargo.toml | 1 + crates/spark/Cargo.toml | 1 + crates/spark/src/function/string/mod.rs | 3 + .../src/function/string/regexp_extract.rs | 245 ++++++++++++++++++ .../test_files/string/regexp_extract.slt | 114 ++++++++ 6 files changed, 365 insertions(+) create mode 100644 crates/spark/src/function/string/regexp_extract.rs create mode 100644 crates/sqllogictest/test_files/string/regexp_extract.slt diff --git a/Cargo.lock b/Cargo.lock index 5f1a84d1..df3967a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1612,6 +1612,7 @@ dependencies = [ "num-traits", "percent-encoding", "rand 0.9.5", + "regex", "serde_json", "sha1", "sha2", diff --git a/Cargo.toml b/Cargo.toml index a4226ef3..5570728c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ datafusion-session = { git = "https://github.com/apache/datafusion", branch = "b log = "^0.4" num-traits = { version = "0.2" } rand = "0.9" +regex = "1" serde_json = "1" sha2 = "^0.11.0" tokio = { version = "1.52", features = ["macros", "rt", "sync"] } diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index 3c82dbb9..0a16332e 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -49,6 +49,7 @@ log = { workspace = true } num-traits = { workspace = true } percent-encoding = "2.3.2" rand = { workspace = true } +regex = { workspace = true } serde_json = { workspace = true } sha1 = "0.11" sha2 = { workspace = true } diff --git a/crates/spark/src/function/string/mod.rs b/crates/spark/src/function/string/mod.rs index 2aef8e34..7aeac67b 100644 --- a/crates/spark/src/function/string/mod.rs +++ b/crates/spark/src/function/string/mod.rs @@ -17,6 +17,7 @@ pub mod like; pub mod luhn_check; pub mod make_valid_utf8; pub mod quote; +pub mod regexp_extract; pub mod soundex; pub mod space; pub mod substring; @@ -45,6 +46,7 @@ make_udf_function!(soundex::SparkSoundex, soundex); make_udf_function!(make_valid_utf8::SparkMakeValidUtf8, make_valid_utf8); make_udf_function!(is_valid_utf8::SparkIsValidUtf8, is_valid_utf8); make_udf_function!(quote::SparkQuote, quote); +make_udf_function!(regexp_extract::SparkRegexpExtract, regexp_extract); make_udf_function!(to_binary::SparkToBinary, to_binary); make_udf_function!(to_binary::SparkTryToBinary, try_to_binary); @@ -171,6 +173,7 @@ pub fn functions() -> Vec> { make_valid_utf8(), is_valid_utf8(), quote(), + regexp_extract(), to_binary(), try_to_binary(), ] diff --git a/crates/spark/src/function/string/regexp_extract.rs b/crates/spark/src/function/string/regexp_extract.rs new file mode 100644 index 00000000..23a4e650 --- /dev/null +++ b/crates/spark/src/function/string/regexp_extract.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder}; +use arrow::datatypes::DataType; +use datafusion_common::cast::as_int32_array; +use datafusion_common::{Result, exec_datafusion_err, exec_err}; +use datafusion_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, +}; +use regex::Regex; + +use crate::function::error_utils::invalid_arg_count_exec_err; + +const NAME: &str = "regexp_extract"; + +/// Spark-compatible `regexp_extract(str, regexp[, idx])`. +/// +/// +/// Returns the text matched by group `idx` (default `1`, `0` is the whole match) +/// of the first match of `regexp` in `str`. +/// +/// - NULL if any argument is NULL. +/// - An empty string if there is no match, or if the group did not take part in +/// the match. +/// - An error if `idx` is outside `0..=groupCount` +/// (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`) or `regexp` is not a valid +/// pattern (`INVALID_PARAMETER_VALUE.PATTERN`). +/// +/// Patterns use the Rust [`regex`] syntax, not `java.util.regex`. Java-only +/// features (lookaround, backreferences, possessive quantifiers) are rejected +/// as invalid patterns here, and shorthands such as `\d` match Unicode digits +/// where Java's match only ASCII. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkRegexpExtract { + signature: Signature, +} + +impl Default for SparkRegexpExtract { + fn default() -> Self { + Self::new() + } +} + +impl SparkRegexpExtract { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkRegexpExtract { + fn name(&self) -> &str { + NAME + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Utf8) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + if !(2..=3).contains(&arg_types.len()) { + return Err(invalid_arg_count_exec_err(NAME, (2, 3), arg_types.len())); + } + let is_text = |t: &DataType| { + matches!( + t, + DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View + | DataType::Null + ) + }; + if !is_text(&arg_types[0]) || !is_text(&arg_types[1]) { + return exec_err!( + "{NAME} expects string str and regexp, got {} and {}", + arg_types[0], + arg_types[1] + ); + } + let mut coerced = vec![DataType::Utf8, DataType::Utf8]; + if let Some(idx) = arg_types.get(2) { + if !idx.is_integer() && *idx != DataType::Null { + return exec_err!("{NAME} expects an INT idx, got {idx}"); + } + coerced.push(DataType::Int32); + } + Ok(coerced) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let num_rows = args.number_rows; + let arrays = ColumnarValue::values_to_arrays(&args.args)?; + let strings = arrays[0].as_string::(); + let patterns = arrays[1].as_string::(); + let default_idx; + let indexes = match arrays.get(2) { + Some(idx) => as_int32_array(idx)?, + None => { + default_idx = Int32Array::from(vec![1; num_rows]); + &default_idx + } + }; + Ok(ColumnarValue::Array(extract(strings, patterns, indexes)?)) + } +} + +fn extract( + strings: &StringArray, + patterns: &StringArray, + indexes: &Int32Array, +) -> Result { + let mut builder = StringBuilder::with_capacity(strings.len(), 0); + // The pattern is almost always a literal, so keep only the last one compiled. + let mut cached: Option<(&str, Regex)> = None; + + for row in 0..strings.len() { + if strings.is_null(row) || patterns.is_null(row) || indexes.is_null(row) { + builder.append_null(); + continue; + } + let pattern = patterns.value(row); + let regex = match &cached { + Some((p, regex)) if *p == pattern => regex, + _ => &cached.insert((pattern, compile(pattern)?)).1, + }; + let idx = indexes.value(row); + let group_count = regex.captures_len() - 1; + let group = usize::try_from(idx).ok().filter(|g| *g <= group_count); + let Some(group) = group else { + return Err(exec_datafusion_err!( + "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of parameter(s) \ + `idx` in `{NAME}` is invalid: Expects group index between 0 and \ + {group_count}, but got {idx}." + )); + }; + let subject = strings.value(row); + let text = regex + .captures(subject) + .and_then(|caps| caps.get(group)) + .map_or("", |m| m.as_str()); + builder.append_value(text); + } + Ok(Arc::new(builder.finish())) +} + +fn compile(pattern: &str) -> Result { + Regex::new(pattern).map_err(|_| { + exec_datafusion_err!( + "[INVALID_PARAMETER_VALUE.PATTERN] The value of parameter(s) `regexp` in \ + `{NAME}` is invalid: '{pattern}'." + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run( + strings: Vec>, + patterns: Vec>, + indexes: Vec>, + ) -> Result>> { + let out = extract( + &StringArray::from(strings), + &StringArray::from(patterns), + &Int32Array::from(indexes), + )?; + Ok(out + .as_string::() + .iter() + .map(|s| s.map(str::to_string)) + .collect()) + } + + fn one(s: &str, p: &str, idx: i32) -> Result { + Ok(run(vec![Some(s)], vec![Some(p)], vec![Some(idx)])? + .remove(0) + .unwrap()) + } + + // Expected values come from Spark's regexp-functions.sql.out golden file. + #[test] + fn groups() -> Result<()> { + assert_eq!(one("1a 2b 14m", r"\d+", 0)?, "1"); + assert_eq!(one("1a 2b 14m", r"(\d+)([a-z]+)", 0)?, "1a"); + assert_eq!(one("1a 2b 14m", r"(\d+)([a-z]+)", 1)?, "1"); + assert_eq!(one("1a 2b 14m", r"(\d+)([a-z]+)", 2)?, "a"); + Ok(()) + } + + #[test] + fn unmatched_group_and_no_match_give_empty_string() -> Result<()> { + assert_eq!(one("1a 2b 14m", r"(\d+)?", 1)?, "1"); + assert_eq!(one("a b m", r"(\d+)?", 1)?, ""); + assert_eq!(one("a b m", r"(\d+)?([a-z]+)", 1)?, ""); + assert_eq!(one("abc", r"(\d+)", 1)?, ""); + Ok(()) + } + + #[test] + fn invalid_group_index_and_pattern() { + for (p, idx) in [(r"\d+", 1), (r"\d+", 2), (r"\d+", -1), (r"(\d+)(a)", 3)] { + let err = one("1a 2b 14m", p, idx).unwrap_err().to_string(); + assert!(err.contains("REGEX_GROUP_INDEX"), "{p} {idx}: {err}"); + } + let err = one("x", "(unclosed", 0).unwrap_err().to_string(); + assert!(err.contains("INVALID_PARAMETER_VALUE.PATTERN"), "{err}"); + } + + #[test] + fn nulls_and_per_row_patterns() -> Result<()> { + let got = run( + vec![Some("a1"), None, Some("b2"), Some("c3"), Some("d4")], + vec![ + Some(r"(\d)"), + Some(r"(\d)"), + None, + Some(r"([a-z])"), + Some(r"(\d)"), + ], + vec![Some(1), Some(1), Some(1), Some(1), None], + )?; + let expected = [Some("1"), None, None, Some("c"), None]; + assert_eq!(got, expected.map(|s| s.map(str::to_string))); + Ok(()) + } + + #[test] + fn null_pattern_skips_validation() -> Result<()> { + // Spark returns NULL before looking at an out-of-range idx on a NULL row. + let got = run(vec![Some("a")], vec![None], vec![Some(99)])?; + assert_eq!(got, vec![None]); + Ok(()) + } +} diff --git a/crates/sqllogictest/test_files/string/regexp_extract.slt b/crates/sqllogictest/test_files/string/regexp_extract.slt new file mode 100644 index 00000000..6c15ef9b --- /dev/null +++ b/crates/sqllogictest/test_files/string/regexp_extract.slt @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible regexp_extract +# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract +# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out. +# Patterns use Rust regex syntax, so Java-only features are not covered. + +query T +SELECT regexp_extract('1a 2b 14m', '\d+', 0); +---- +1 + +# Default group index is 1 +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)'); +---- +1 + +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0); +---- +1a + +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 1); +---- +1 + +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2); +---- +a + +# A group that did not take part in the match, or no match at all, gives '' +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)?', 1); +---- +1 + +query T +SELECT regexp_extract('a b m', '(\d+)?', 1); +---- +(empty) + +query T +SELECT regexp_extract('1a 2b 14m', '(\d+)?([a-z]+)', 1); +---- +1 + +query T +SELECT regexp_extract('a b m', '(\d+)?([a-z]+)', 1); +---- +(empty) + +query T +SELECT regexp_extract('abc', '(\d+)', 1); +---- +(empty) + +# NULL in any argument gives NULL +query T +SELECT regexp_extract(NULL, '(\d+)', 1); +---- +NULL + +query T +SELECT regexp_extract('a1', NULL, 1); +---- +NULL + +query T +SELECT regexp_extract('a1', '(\d+)', NULL); +---- +NULL + +# Group index out of range +statement error INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX.*Expects group index between 0 and 0, but got 1 +SELECT regexp_extract('1a 2b 14m', '\d+'); + +statement error Expects group index between 0 and 0, but got 1 +SELECT regexp_extract('1a 2b 14m', '\d+', 1); + +statement error Expects group index between 0 and 0, but got 2 +SELECT regexp_extract('1a 2b 14m', '\d+', 2); + +statement error Expects group index between 0 and 0, but got -1 +SELECT regexp_extract('1a 2b 14m', '\d+', -1); + +statement error Expects group index between 0 and 2, but got 3 +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 3); + +statement error Expects group index between 0 and 2, but got -1 +SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1); + +# Invalid pattern +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('1a 2b 14m', '(?l)'); + +# Wrong number of arguments +statement error +SELECT regexp_extract('abc'); + +# Column input, including NULL rows and a per-row pattern +statement ok +CREATE TABLE regexp_extract_t AS VALUES ('100-200', '(\d+)-(\d+)'), (NULL, '(\d+)'), ('abc', '([a-z])'); + +query T +SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_t; +---- +100 +NULL +a From 6372e0bdc98efcaf961755e3966f9883854e8c00 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 09:15:46 -0700 Subject: [PATCH 2/8] test(spark): extend regexp_extract slt coverage --- .../test_files/string/regexp_extract.slt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/sqllogictest/test_files/string/regexp_extract.slt b/crates/sqllogictest/test_files/string/regexp_extract.slt index 6c15ef9b..9db07867 100644 --- a/crates/sqllogictest/test_files/string/regexp_extract.slt +++ b/crates/sqllogictest/test_files/string/regexp_extract.slt @@ -112,3 +112,21 @@ SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_t; 100 NULL a + +# Other string types are accepted +query T +SELECT regexp_extract(arrow_cast('1a 2b', 'LargeUtf8'), arrow_cast('(\d+)([a-z])', 'Utf8View'), 2); +---- +a + +# Integer types other than INT are accepted for idx +query T +SELECT regexp_extract('1a 2b', '(\d+)([a-z])', arrow_cast(2, 'Int8')); +---- +a + +# The result type is a string +query T +SELECT arrow_typeof(regexp_extract('a', '(a)')); +---- +Utf8 From 4bfa9f2295ad29fd360a8830162fa2e3150f2ccc Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 09:16:46 -0700 Subject: [PATCH 3/8] perf(spark): reuse capture locations across rows in regexp_extract --- .../src/function/string/regexp_extract.rs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/spark/src/function/string/regexp_extract.rs b/crates/spark/src/function/string/regexp_extract.rs index 23a4e650..c4a1ca33 100644 --- a/crates/spark/src/function/string/regexp_extract.rs +++ b/crates/spark/src/function/string/regexp_extract.rs @@ -11,7 +11,7 @@ use datafusion_common::{Result, exec_datafusion_err, exec_err}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use regex::Regex; +use regex::{CaptureLocations, Regex}; use crate::function::error_utils::invalid_arg_count_exec_err; @@ -119,8 +119,9 @@ fn extract( indexes: &Int32Array, ) -> Result { let mut builder = StringBuilder::with_capacity(strings.len(), 0); - // The pattern is almost always a literal, so keep only the last one compiled. - let mut cached: Option<(&str, Regex)> = None; + // The pattern is almost always a literal, so keep only the last one compiled, + // with a capture buffer reused across rows instead of allocated per match. + let mut cached: Option<(&str, Regex, CaptureLocations)> = None; for row in 0..strings.len() { if strings.is_null(row) || patterns.is_null(row) || indexes.is_null(row) { @@ -128,9 +129,13 @@ fn extract( continue; } let pattern = patterns.value(row); - let regex = match &cached { - Some((p, regex)) if *p == pattern => regex, - _ => &cached.insert((pattern, compile(pattern)?)).1, + let (_, regex, locations) = match &mut cached { + Some(entry) if entry.0 == pattern => entry, + _ => { + let regex = compile(pattern)?; + let locations = regex.capture_locations(); + cached.insert((pattern, regex, locations)) + } }; let idx = indexes.value(row); let group_count = regex.captures_len() - 1; @@ -144,9 +149,9 @@ fn extract( }; let subject = strings.value(row); let text = regex - .captures(subject) - .and_then(|caps| caps.get(group)) - .map_or("", |m| m.as_str()); + .captures_read(locations, subject) + .and_then(|_| locations.get(group)) + .map_or("", |(start, end)| &subject[start..end]); builder.append_value(text); } Ok(Arc::new(builder.finish())) From d57453e3f95add44146ae65108d812affaeab48f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 09:51:53 -0700 Subject: [PATCH 4/8] fix(spark): address regexp_extract review feedback - Check idx only when the pattern matches, as Spark does, so a non-matching row returns '' even when idx is out of range. - Keep a literal pattern and idx as scalars: compile the pattern once and avoid per-row copies; return a scalar when all arguments are scalars. Reserve output data capacity from the input string bytes. - Export regexp_extract in expr_fn. - Move the Rust unit tests to the .slt file, with expected values verified against Spark 4.2.0. Co-Authored-By: Claude Opus 5.5 --- crates/spark/src/function/string/mod.rs | 5 + .../src/function/string/regexp_extract.rs | 238 +++++++++--------- .../test_files/string/regexp_extract.slt | 99 +++++++- 3 files changed, 219 insertions(+), 123 deletions(-) diff --git a/crates/spark/src/function/string/mod.rs b/crates/spark/src/function/string/mod.rs index 7aeac67b..0466e3c6 100644 --- a/crates/spark/src/function/string/mod.rs +++ b/crates/spark/src/function/string/mod.rs @@ -140,6 +140,11 @@ pub mod expr_fn { "Returns str enclosed by single quotes and each instance of single quote in it is preceded by a backslash", str )); + export_functions!(( + regexp_extract, + "Extracts the group `idx` (default 1, 0 for the whole match) of the first match of `regexp` in `str`.", + str regexp idx + )); export_functions!(( to_binary, "Converts the input str to a binary value based on the supplied fmt, which must be a case-insensitive literal of 'hex', 'utf-8', 'utf8' or 'base64'. Defaults to 'hex'.", diff --git a/crates/spark/src/function/string/regexp_extract.rs b/crates/spark/src/function/string/regexp_extract.rs index c4a1ca33..7eb25e83 100644 --- a/crates/spark/src/function/string/regexp_extract.rs +++ b/crates/spark/src/function/string/regexp_extract.rs @@ -4,10 +4,12 @@ use std::sync::Arc; -use arrow::array::{Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder}; +use arrow::array::{ + Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, new_null_array, +}; use arrow::datatypes::DataType; use datafusion_common::cast::as_int32_array; -use datafusion_common::{Result, exec_datafusion_err, exec_err}; +use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; @@ -97,66 +99,141 @@ impl ScalarUDFImpl for SparkRegexpExtract { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let num_rows = args.number_rows; - let arrays = ColumnarValue::values_to_arrays(&args.args)?; - let strings = arrays[0].as_string::(); - let patterns = arrays[1].as_string::(); - let default_idx; - let indexes = match arrays.get(2) { - Some(idx) => as_int32_array(idx)?, - None => { - default_idx = Int32Array::from(vec![1; num_rows]); - &default_idx + let all_scalar = args + .args + .iter() + .all(|arg| matches!(arg, ColumnarValue::Scalar(_))); + let len = if all_scalar { 1 } else { args.number_rows }; + + // The usual call has a literal pattern and idx: keep those as scalars so + // the pattern is compiled once and no per-row copies are allocated. + let pattern = match &args.args[1] { + ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? { + Some(pattern) => Pattern::Scalar(pattern), + None => return null_result(len, all_scalar), + }, + ColumnarValue::Array(array) => Pattern::Array(array.as_string::()), + }; + let index = match args.args.get(2) { + None => Index::Scalar(1), + Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => { + Index::Scalar(*idx) + } + Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | ScalarValue::Null)) => { + return null_result(len, all_scalar); } + Some(ColumnarValue::Scalar(other)) => { + return exec_err!("{NAME} expects an INT idx, got {}", other.data_type()); + } + Some(ColumnarValue::Array(array)) => Index::Array(as_int32_array(array)?), }; - Ok(ColumnarValue::Array(extract(strings, patterns, indexes)?)) + + let strings = args.args[0].to_array(len)?; + let result = extract(strings.as_string::(), pattern, index)?; + if all_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } +} + +#[derive(Clone, Copy)] +enum Pattern<'a> { + Scalar(&'a str), + Array(&'a StringArray), +} + +#[derive(Clone, Copy)] +enum Index<'a> { + Scalar(i32), + Array(&'a Int32Array), +} + +fn scalar_str(scalar: &ScalarValue) -> Result> { + match scalar { + ScalarValue::Null => Ok(None), + other => other.try_as_str().ok_or_else(|| { + exec_datafusion_err!( + "{NAME} expects a string regexp, got {}", + other.data_type() + ) + }), } } -fn extract( - strings: &StringArray, - patterns: &StringArray, - indexes: &Int32Array, -) -> Result { - let mut builder = StringBuilder::with_capacity(strings.len(), 0); - // The pattern is almost always a literal, so keep only the last one compiled, - // with a capture buffer reused across rows instead of allocated per match. +fn null_result(len: usize, all_scalar: bool) -> Result { + Ok(if all_scalar { + ColumnarValue::Scalar(ScalarValue::Utf8(None)) + } else { + ColumnarValue::Array(new_null_array(&DataType::Utf8, len)) + }) +} + +fn extract(strings: &StringArray, pattern: Pattern, index: Index) -> Result { + // Each result is a substring of its input, so the input bytes bound the output. + let mut builder = + StringBuilder::with_capacity(strings.len(), strings.value_data().len()); + // A scalar pattern is compiled on the first non-NULL row and reused. A column + // pattern keeps only the last one compiled, since it rarely changes per row. + // The capture buffer is reused across rows instead of allocated per match. let mut cached: Option<(&str, Regex, CaptureLocations)> = None; + let scalar_pattern = matches!(pattern, Pattern::Scalar(_)); + for row in 0..strings.len() { - if strings.is_null(row) || patterns.is_null(row) || indexes.is_null(row) { + let row_pattern = match pattern { + Pattern::Scalar(pattern) => Some(pattern), + Pattern::Array(patterns) => { + (!patterns.is_null(row)).then(|| patterns.value(row)) + } + }; + let idx = match index { + Index::Scalar(idx) => Some(idx), + Index::Array(indexes) => (!indexes.is_null(row)).then(|| indexes.value(row)), + }; + let (false, Some(pattern), Some(idx)) = (strings.is_null(row), row_pattern, idx) + else { builder.append_null(); continue; - } - let pattern = patterns.value(row); + }; let (_, regex, locations) = match &mut cached { - Some(entry) if entry.0 == pattern => entry, + Some(entry) if scalar_pattern || entry.0 == pattern => entry, _ => { let regex = compile(pattern)?; let locations = regex.capture_locations(); cached.insert((pattern, regex, locations)) } }; - let idx = indexes.value(row); - let group_count = regex.captures_len() - 1; - let group = usize::try_from(idx).ok().filter(|g| *g <= group_count); - let Some(group) = group else { - return Err(exec_datafusion_err!( - "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of parameter(s) \ - `idx` in `{NAME}` is invalid: Expects group index between 0 and \ - {group_count}, but got {idx}." - )); - }; let subject = strings.value(row); - let text = regex - .captures_read(locations, subject) - .and_then(|_| locations.get(group)) - .map_or("", |(start, end)| &subject[start..end]); + // Like Spark, idx is only checked when the pattern matches. + let text = match regex.captures_read(locations, subject) { + None => "", + Some(_) => locations + .get(check_group_index(regex, idx)?) + .map_or("", |(start, end)| &subject[start..end]), + }; builder.append_value(text); } Ok(Arc::new(builder.finish())) } +fn check_group_index(regex: &Regex, idx: i32) -> Result { + let group_count = regex.captures_len() - 1; + usize::try_from(idx) + .ok() + .filter(|group| *group <= group_count) + .ok_or_else(|| { + exec_datafusion_err!( + "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of parameter(s) \ + `idx` in `{NAME}` is invalid: Expects group index between 0 and \ + {group_count}, but got {idx}." + ) + }) +} + fn compile(pattern: &str) -> Result { Regex::new(pattern).map_err(|_| { exec_datafusion_err!( @@ -165,86 +242,3 @@ fn compile(pattern: &str) -> Result { ) }) } - -#[cfg(test)] -mod tests { - use super::*; - - fn run( - strings: Vec>, - patterns: Vec>, - indexes: Vec>, - ) -> Result>> { - let out = extract( - &StringArray::from(strings), - &StringArray::from(patterns), - &Int32Array::from(indexes), - )?; - Ok(out - .as_string::() - .iter() - .map(|s| s.map(str::to_string)) - .collect()) - } - - fn one(s: &str, p: &str, idx: i32) -> Result { - Ok(run(vec![Some(s)], vec![Some(p)], vec![Some(idx)])? - .remove(0) - .unwrap()) - } - - // Expected values come from Spark's regexp-functions.sql.out golden file. - #[test] - fn groups() -> Result<()> { - assert_eq!(one("1a 2b 14m", r"\d+", 0)?, "1"); - assert_eq!(one("1a 2b 14m", r"(\d+)([a-z]+)", 0)?, "1a"); - assert_eq!(one("1a 2b 14m", r"(\d+)([a-z]+)", 1)?, "1"); - assert_eq!(one("1a 2b 14m", r"(\d+)([a-z]+)", 2)?, "a"); - Ok(()) - } - - #[test] - fn unmatched_group_and_no_match_give_empty_string() -> Result<()> { - assert_eq!(one("1a 2b 14m", r"(\d+)?", 1)?, "1"); - assert_eq!(one("a b m", r"(\d+)?", 1)?, ""); - assert_eq!(one("a b m", r"(\d+)?([a-z]+)", 1)?, ""); - assert_eq!(one("abc", r"(\d+)", 1)?, ""); - Ok(()) - } - - #[test] - fn invalid_group_index_and_pattern() { - for (p, idx) in [(r"\d+", 1), (r"\d+", 2), (r"\d+", -1), (r"(\d+)(a)", 3)] { - let err = one("1a 2b 14m", p, idx).unwrap_err().to_string(); - assert!(err.contains("REGEX_GROUP_INDEX"), "{p} {idx}: {err}"); - } - let err = one("x", "(unclosed", 0).unwrap_err().to_string(); - assert!(err.contains("INVALID_PARAMETER_VALUE.PATTERN"), "{err}"); - } - - #[test] - fn nulls_and_per_row_patterns() -> Result<()> { - let got = run( - vec![Some("a1"), None, Some("b2"), Some("c3"), Some("d4")], - vec![ - Some(r"(\d)"), - Some(r"(\d)"), - None, - Some(r"([a-z])"), - Some(r"(\d)"), - ], - vec![Some(1), Some(1), Some(1), Some(1), None], - )?; - let expected = [Some("1"), None, None, Some("c"), None]; - assert_eq!(got, expected.map(|s| s.map(str::to_string))); - Ok(()) - } - - #[test] - fn null_pattern_skips_validation() -> Result<()> { - // Spark returns NULL before looking at an out-of-range idx on a NULL row. - let got = run(vec![Some("a")], vec![None], vec![Some(99)])?; - assert_eq!(got, vec![None]); - Ok(()) - } -} diff --git a/crates/sqllogictest/test_files/string/regexp_extract.slt b/crates/sqllogictest/test_files/string/regexp_extract.slt index 9db07867..80caa8e7 100644 --- a/crates/sqllogictest/test_files/string/regexp_extract.slt +++ b/crates/sqllogictest/test_files/string/regexp_extract.slt @@ -4,7 +4,8 @@ # Tests for Spark-compatible regexp_extract # Reference: https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract -# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out. +# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out +# or were verified against Spark 4.2.0. # Patterns use Rust regex syntax, so Java-only features are not covered. query T @@ -59,6 +60,33 @@ SELECT regexp_extract('abc', '(\d+)', 1); ---- (empty) +# idx is only checked when the pattern matches, so no match gives '' even +# when idx is out of range (including the default idx 1 with no groups) +query T +SELECT regexp_extract('abc', '\d+'); +---- +(empty) + +query T +SELECT regexp_extract('abc', '\d+', 5); +---- +(empty) + +query T +SELECT regexp_extract('abc', '\d+', -1); +---- +(empty) + +query T +SELECT regexp_extract('abc', '(\d+)', 2); +---- +(empty) + +query T +SELECT regexp_extract('', '\d+', 3); +---- +(empty) + # NULL in any argument gives NULL query T SELECT regexp_extract(NULL, '(\d+)', 1); @@ -75,6 +103,12 @@ SELECT regexp_extract('a1', '(\d+)', NULL); ---- NULL +# A NULL row returns NULL before idx is checked +query T +SELECT regexp_extract('a', NULL, 99); +---- +NULL + # Group index out of range statement error INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX.*Expects group index between 0 and 0, but got 1 SELECT regexp_extract('1a 2b 14m', '\d+'); @@ -98,6 +132,9 @@ SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1); statement error INVALID_PARAMETER_VALUE.PATTERN SELECT regexp_extract('1a 2b 14m', '(?l)'); +statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed' +SELECT regexp_extract('x', '(unclosed', 0); + # Wrong number of arguments statement error SELECT regexp_extract('abc'); @@ -113,6 +150,57 @@ SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_t; NULL a +# Per-row patterns alternating between rows, with NULL string and pattern rows +statement ok +CREATE TABLE regexp_extract_patterns AS VALUES ('a1', '(\d)'), (NULL, '(\d)'), ('b2', NULL), ('c3', '([a-z])'), ('d4', '(\d)'), ('e5', '([a-z])(\d)'); + +query T +SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_patterns; +---- +1 +NULL +NULL +c +4 +e + +# Per-row idx, including NULL and an out-of-range idx on a row that does not match +statement ok +CREATE TABLE regexp_extract_idx AS VALUES ('a1', '(\d)', 1), ('d4', '(\d)', NULL), ('xy', '(\d)', 7), ('c3', '([a-z])(\d)', 2); + +query T +SELECT regexp_extract(column1, column2, column3) FROM regexp_extract_idx; +---- +1 +NULL +(empty) +3 + +query T +SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), ('def', -1), (NULL, 9)); +---- +(empty) +(empty) +NULL + +# An out-of-range idx fails once a row matches +statement error Expects group index between 0 and 1, but got 3 +SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), ('12', 3)); + +# Literal pattern with no match on any row and the default idx +query T +SELECT regexp_extract(column1, '\d+') FROM (VALUES ('abc'), ('xyz')); +---- +(empty) +(empty) + +# An invalid literal pattern is not compiled when every string is NULL +query T +SELECT regexp_extract(column1, '(unclosed', 0) FROM (VALUES (CAST(NULL AS STRING)), (CAST(NULL AS STRING))); +---- +NULL +NULL + # Other string types are accepted query T SELECT regexp_extract(arrow_cast('1a 2b', 'LargeUtf8'), arrow_cast('(\d+)([a-z])', 'Utf8View'), 2); @@ -130,3 +218,12 @@ query T SELECT arrow_typeof(regexp_extract('a', '(a)')); ---- Utf8 + +statement ok +DROP TABLE regexp_extract_t; + +statement ok +DROP TABLE regexp_extract_patterns; + +statement ok +DROP TABLE regexp_extract_idx; From d55c7fc9a31534d27b132a9f480592fcf1307765 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 14:21:20 -0700 Subject: [PATCH 5/8] docs(spark): document regexp_extract with #[user_doc] Co-Authored-By: Claude Opus 5.5 --- .../src/function/string/regexp_extract.rs | 62 ++++++++++++++----- .../test_files/string/regexp_extract.slt | 16 ++++- 2 files changed, 60 insertions(+), 18 deletions(-) diff --git a/crates/spark/src/function/string/regexp_extract.rs b/crates/spark/src/function/string/regexp_extract.rs index 7eb25e83..5f6aa2f2 100644 --- a/crates/spark/src/function/string/regexp_extract.rs +++ b/crates/spark/src/function/string/regexp_extract.rs @@ -11,31 +11,55 @@ use arrow::datatypes::DataType; use datafusion_common::cast::as_int32_array; use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err}; use datafusion_expr::{ - ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, }; +use datafusion_macros::user_doc; use regex::{CaptureLocations, Regex}; use crate::function::error_utils::invalid_arg_count_exec_err; const NAME: &str = "regexp_extract"; -/// Spark-compatible `regexp_extract(str, regexp[, idx])`. -/// -/// -/// Returns the text matched by group `idx` (default `1`, `0` is the whole match) -/// of the first match of `regexp` in `str`. -/// -/// - NULL if any argument is NULL. -/// - An empty string if there is no match, or if the group did not take part in -/// the match. -/// - An error if `idx` is outside `0..=groupCount` -/// (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`) or `regexp` is not a valid -/// pattern (`INVALID_PARAMETER_VALUE.PATTERN`). +/// Spark-compatible `regexp_extract` expression. /// -/// Patterns use the Rust [`regex`] syntax, not `java.util.regex`. Java-only -/// features (lookaround, backreferences, possessive quantifiers) are rejected -/// as invalid patterns here, and shorthands such as `\d` match Unicode digits -/// where Java's match only ASCII. +/// +#[user_doc( + doc_section(label = "Regular Expression Functions"), + description = "Returns the text matched by group `idx` of the first match of `regexp` in `str`. \ + Unlike DataFusion's `regexp_match`, which returns a list of every capture group, this returns a single string. \ + Returns an empty string if there is no match, or if the group did not take part in the match. \ + `idx` is only checked when the pattern matches: an `idx` outside `0..=groupCount` is an error \ + (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`) on a row that matches, and an empty string on a row that does not. \ + An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). Returns NULL if any argument is NULL. \ + Patterns use Rust `regex` syntax, not Java's `java.util.regex` as in Spark: Java-only features such as \ + lookaround and backreferences are rejected as invalid patterns, and `\\d` matches any Unicode digit, not just ASCII 0-9.", + syntax_example = "regexp_extract(str, regexp[, idx])", + sql_example = r#"```sql +> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)'); +1 +> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0); +1a +> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2); +a +> SELECT regexp_extract('abc', '(\d+)', 1); +-- empty string +> SELECT regexp_extract('a1', '(\d+)', NULL); +NULL +```"#, + argument( + name = "str", + description = "String expression to search. NULL returns NULL." + ), + argument( + name = "regexp", + description = "String regular expression in Rust `regex` syntax. NULL returns NULL." + ), + argument( + name = "idx", + description = "Optional integer capture group index. Defaults to 1; 0 is the whole match. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkRegexpExtract { signature: Signature, @@ -68,6 +92,10 @@ impl ScalarUDFImpl for SparkRegexpExtract { Ok(DataType::Utf8) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { if !(2..=3).contains(&arg_types.len()) { return Err(invalid_arg_count_exec_err(NAME, (2, 3), arg_types.len())); diff --git a/crates/sqllogictest/test_files/string/regexp_extract.slt b/crates/sqllogictest/test_files/string/regexp_extract.slt index 80caa8e7..ed97b3d8 100644 --- a/crates/sqllogictest/test_files/string/regexp_extract.slt +++ b/crates/sqllogictest/test_files/string/regexp_extract.slt @@ -6,7 +6,8 @@ # Reference: https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract # Expected values come from Spark's sql-tests/results/regexp-functions.sql.out # or were verified against Spark 4.2.0. -# Patterns use Rust regex syntax, so Java-only features are not covered. +# Patterns use Rust regex syntax, so Java-only features are not covered +# beyond checking that they are rejected. query T SELECT regexp_extract('1a 2b 14m', '\d+', 0); @@ -135,6 +136,19 @@ SELECT regexp_extract('1a 2b 14m', '(?l)'); statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed' SELECT regexp_extract('x', '(unclosed', 0); +# Rust regex syntax: Java-only lookaround and backreferences are invalid patterns +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('ab', 'a(?=b)', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('aa', '(a)\1', 0); + +# Rust regex \d matches any Unicode digit (Spark's Java \d matches only ASCII 0-9) +query T +SELECT regexp_extract('x٣y', '\d', 0); +---- +٣ + # Wrong number of arguments statement error SELECT regexp_extract('abc'); From 78d02c9743d3cb276399afb68f362bca496f242a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 11:30:00 -0700 Subject: [PATCH 6/8] fix(spark): align regexp_extract coercion and errors with Spark - Cast str and regexp of other types to strings, and numeric or string idx to INT, as Spark's implicit casts do; type errors are now plan errors. - Report look-around, backreferences and patterns over the Rust compiled size limit as unsupported (not implemented) instead of INVALID_PARAMETER_VALUE.PATTERN, since Spark accepts them. - Quote the pattern in the PATTERN error like Spark's toSQLValue. - Document the Rust/Java regex differences and record each as a commented-out query with Spark's result (#255). - Add tests for literal/column argument mixes, per-row invalid patterns, the empty pattern and coercion. Co-Authored-By: Claude Opus 5.5 --- .../src/function/string/regexp_extract.rs | 115 ++++++++--- .../test_files/string/regexp_extract.slt | 189 ++++++++++++++++-- 2 files changed, 260 insertions(+), 44 deletions(-) diff --git a/crates/spark/src/function/string/regexp_extract.rs b/crates/spark/src/function/string/regexp_extract.rs index 5f6aa2f2..3fdc00b6 100644 --- a/crates/spark/src/function/string/regexp_extract.rs +++ b/crates/spark/src/function/string/regexp_extract.rs @@ -7,9 +7,13 @@ use std::sync::Arc; use arrow::array::{ Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, new_null_array, }; +use arrow::compute::can_cast_types; use arrow::datatypes::DataType; use datafusion_common::cast::as_int32_array; -use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err}; +use datafusion_common::{ + DataFusionError, Result, ScalarValue, exec_datafusion_err, internal_datafusion_err, + internal_err, plan_err, +}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -30,10 +34,21 @@ const NAME: &str = "regexp_extract"; Unlike DataFusion's `regexp_match`, which returns a list of every capture group, this returns a single string. \ Returns an empty string if there is no match, or if the group did not take part in the match. \ `idx` is only checked when the pattern matches: an `idx` outside `0..=groupCount` is an error \ - (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`) on a row that matches, and an empty string on a row that does not. \ - An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). Returns NULL if any argument is NULL. \ - Patterns use Rust `regex` syntax, not Java's `java.util.regex` as in Spark: Java-only features such as \ - lookaround and backreferences are rejected as invalid patterns, and `\\d` matches any Unicode digit, not just ASCII 0-9.", + (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and later; Spark 3.4 raised \ + `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty string on a row that does not. \ + An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). Returns NULL if any argument is NULL, \ + and a NULL row is not checked, so an invalid pattern is only reported on a row it is applied to. \ + As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \ + Known differences from Spark (https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \ + `regex` engine, not Java's `java.util.regex`. Look-around, backreferences, and patterns that exceed the Rust \ + engine's compiled-size limit raise an \"unsupported regex pattern\" error, where Spark evaluates them. \ + Possessive quantifiers are read as nested greedy repetition (`a++` as `(a+)+`), so they can match \ + where Java's would not. \ + `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match only ASCII. \ + `.` matches `\\r`, where Java's does not. `$` matches only at the very end of the input, where Java's also \ + matches before a final line terminator. `(?i)` folds case using Unicode rules (so `k` matches the Kelvin \ + sign), where Java folds only ASCII by default. Rust's `(?P...)` named groups are accepted, where Java \ + rejects them as an invalid pattern.", syntax_example = "regexp_extract(str, regexp[, idx])", sql_example = r#"```sql > SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)'); @@ -49,15 +64,15 @@ NULL ```"#, argument( name = "str", - description = "String expression to search. NULL returns NULL." + description = "String expression to search. Values of other types, such as numbers and binary, are cast to a string. NULL returns NULL." ), argument( name = "regexp", - description = "String regular expression in Rust `regex` syntax. NULL returns NULL." + description = "String regular expression in Rust `regex` syntax. Values of other types are cast to a string. NULL returns NULL." ), argument( name = "idx", - description = "Optional integer capture group index. Defaults to 1; 0 is the whole match. NULL returns NULL." + description = "Optional capture group index, cast to INT (numeric and string values are accepted). Defaults to 1; 0 is the whole match. NULL returns NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -100,26 +115,27 @@ impl ScalarUDFImpl for SparkRegexpExtract { if !(2..=3).contains(&arg_types.len()) { return Err(invalid_arg_count_exec_err(NAME, (2, 3), arg_types.len())); } - let is_text = |t: &DataType| { - matches!( - t, - DataType::Utf8 - | DataType::LargeUtf8 - | DataType::Utf8View - | DataType::Null - ) + // Like Spark's implicit casts: str and regexp are cast to strings, idx to INT. + let castable_to_string = |t: &DataType| { + *t == DataType::Null || (!t.is_nested() && can_cast_types(t, &DataType::Utf8)) }; - if !is_text(&arg_types[0]) || !is_text(&arg_types[1]) { - return exec_err!( - "{NAME} expects string str and regexp, got {} and {}", - arg_types[0], - arg_types[1] - ); + for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) { + if !castable_to_string(arg_type) { + return plan_err!("{NAME} expects a string {name}, got {arg_type}"); + } } let mut coerced = vec![DataType::Utf8, DataType::Utf8]; if let Some(idx) = arg_types.get(2) { - if !idx.is_integer() && *idx != DataType::Null { - return exec_err!("{NAME} expects an INT idx, got {idx}"); + let castable_to_int = idx.is_numeric() + || matches!( + idx, + DataType::Null + | DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View + ); + if !castable_to_int { + return plan_err!("{NAME} expects an INT idx, got {idx}"); } coerced.push(DataType::Int32); } @@ -151,7 +167,10 @@ impl ScalarUDFImpl for SparkRegexpExtract { return null_result(len, all_scalar); } Some(ColumnarValue::Scalar(other)) => { - return exec_err!("{NAME} expects an INT idx, got {}", other.data_type()); + return internal_err!( + "{NAME} expects an INT idx, got {}", + other.data_type() + ); } Some(ColumnarValue::Array(array)) => Index::Array(as_int32_array(array)?), }; @@ -184,7 +203,7 @@ fn scalar_str(scalar: &ScalarValue) -> Result> { match scalar { ScalarValue::Null => Ok(None), other => other.try_as_str().ok_or_else(|| { - exec_datafusion_err!( + internal_datafusion_err!( "{NAME} expects a string regexp, got {}", other.data_type() ) @@ -263,10 +282,44 @@ fn check_group_index(regex: &Regex, idx: i32) -> Result { } fn compile(pattern: &str) -> Result { - Regex::new(pattern).map_err(|_| { - exec_datafusion_err!( - "[INVALID_PARAMETER_VALUE.PATTERN] The value of parameter(s) `regexp` in \ - `{NAME}` is invalid: '{pattern}'." - ) + Regex::new(pattern).map_err(|err| { + let pattern = to_sql_value(pattern); + match unsupported_reason(&err) { + Some(reason) => DataFusionError::NotImplemented(format!( + "Unsupported regex pattern {pattern} in `{NAME}`: {reason}. \ + See https://github.com/apache/datafusion-spark/issues/255" + )), + None => exec_datafusion_err!( + "[INVALID_PARAMETER_VALUE.PATTERN] The value of parameter(s) `regexp` in \ + `{NAME}` is invalid: {pattern}." + ), + } }) } + +/// Returns why a pattern the Rust engine rejects may still be valid in Java, +/// for the errors the `regex` crate reports for features it does not support. +fn unsupported_reason(err: ®ex::Error) -> Option<&'static str> { + match err { + regex::Error::CompiledTooBig(_) => { + Some("the compiled pattern exceeds the Rust regex size limit") + } + regex::Error::Syntax(msg) + if msg.contains( + "look-around, including look-ahead and look-behind, is not supported", + ) => + { + Some("look-around is not supported by the Rust regex engine") + } + regex::Error::Syntax(msg) if msg.contains("backreferences are not supported") => { + Some("backreferences are not supported by the Rust regex engine") + } + _ => None, + } +} + +/// Quotes a string the way Spark's `toSQLValue` renders a string literal in an +/// error message: wrapped in single quotes, with `\` and `'` escaped. +fn to_sql_value(value: &str) -> String { + format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")) +} diff --git a/crates/sqllogictest/test_files/string/regexp_extract.slt b/crates/sqllogictest/test_files/string/regexp_extract.slt index ed97b3d8..430931c8 100644 --- a/crates/sqllogictest/test_files/string/regexp_extract.slt +++ b/crates/sqllogictest/test_files/string/regexp_extract.slt @@ -6,8 +6,13 @@ # Reference: https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract # Expected values come from Spark's sql-tests/results/regexp-functions.sql.out # or were verified against Spark 4.2.0. -# Patterns use Rust regex syntax, so Java-only features are not covered -# beyond checking that they are rejected. +# Audited against Spark 4.2.0 on 2026-11-15. +# Expected values verified with pyspark==4.2.0. +# Known divergences are captured as commented-out queries below. +# Patterns use the Rust regex engine, not java.util.regex; the differences are +# tracked by https://github.com/apache/datafusion-spark/issues/255 +# The REGEX_GROUP_INDEX error matches Spark 3.5 and later (Spark 3.4 raised +# _LEGACY_ERROR_TEMP_2006 / _LEGACY_ERROR_TEMP_2007 instead). query T SELECT regexp_extract('1a 2b 14m', '\d+', 0); @@ -104,7 +109,18 @@ SELECT regexp_extract('a1', '(\d+)', NULL); ---- NULL -# A NULL row returns NULL before idx is checked +# A NULL row returns NULL before idx or the pattern is checked +query T +SELECT regexp_extract(NULL, '(unclosed', 0); +---- +NULL + +query T +SELECT regexp_extract('a', '(unclosed', NULL); +---- +NULL + +# A NULL idx returns NULL before idx is checked query T SELECT regexp_extract('a', NULL, 99); ---- @@ -136,18 +152,88 @@ SELECT regexp_extract('1a 2b 14m', '(?l)'); statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed' SELECT regexp_extract('x', '(unclosed', 0); -# Rust regex syntax: Java-only lookaround and backreferences are invalid patterns -statement error INVALID_PARAMETER_VALUE.PATTERN -SELECT regexp_extract('ab', 'a(?=b)', 0); +# The pattern is quoted like Spark's toSQLValue, with \ and ' escaped +statement error is invalid: '\(\\\\d'\. +SELECT regexp_extract('x', '(\d', 0); -statement error INVALID_PARAMETER_VALUE.PATTERN -SELECT regexp_extract('aa', '(a)\1', 0); +statement error is invalid: 'it\\'s\('\. +SELECT regexp_extract('x', 'it''s(', 0); -# Rust regex \d matches any Unicode digit (Spark's Java \d matches only ASCII 0-9) +# An empty pattern matches the empty string query T -SELECT regexp_extract('x٣y', '\d', 0); +SELECT regexp_extract('abc', '', 0); ---- -٣ +(empty) + +# Known differences between the Rust regex engine and java.util.regex +# https://github.com/apache/datafusion-spark/issues/255 + +# Look-around and backreferences are not supported by the Rust regex engine +statement error Unsupported regex pattern 'a\(\?=b\)' in `regexp_extract`: look-around +SELECT regexp_extract('ab', 'a(?=b)', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: backreferences +SELECT regexp_extract('aa', '(a)\1', 0); + +# Spark 4.2.0 supports look-around +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('ab', 'a(?=b)', 0); +# ---- +# a + +# Spark 4.2.0 supports backreferences +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('aa', '(a)\1', 0); +# ---- +# aa + +# Spark 4.2.0 treats a++ as a possessive quantifier that does not give back, +# so a++a cannot match; the Rust regex engine reads a++ as (a+)+ and returns 'aaa' +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('aaa', '(a++a)', 1); +# ---- +# (empty) + +# A pattern over the Rust regex engine's compiled-size limit is not supported +statement error Unsupported regex pattern .* in `regexp_extract`: the compiled pattern exceeds the Rust regex size limit +SELECT regexp_extract('a', '\w{1000}\w{1000}\w{1000}', 0); + +# Spark 4.2.0 matches \d against ASCII digits only; DataFusion returns '٣' +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('x٣y', '\d', 0); +# ---- +# (empty) + +# Spark 4.2.0's . does not match a carriage return; DataFusion returns 'a\rb' +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('a' || chr(13) || 'b', '(a.b)', 1); +# ---- +# (empty) + +# Spark 4.2.0's $ also matches before a final line terminator; DataFusion returns '' +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('abc' || chr(10), '(abc$)', 1); +# ---- +# abc + +# Spark 4.2.0's (?i) folds ASCII case only, so k does not match the Kelvin sign +# (U+212A); DataFusion returns it +# https://github.com/apache/datafusion-spark/issues/255 +# query T +# SELECT regexp_extract('K', '((?i)k)', 1); +# ---- +# (empty) + +# Spark 4.2.0 rejects Python-style named groups; DataFusion returns 'a' +# https://github.com/apache/datafusion-spark/issues/255 +# statement error INVALID_PARAMETER_VALUE.PATTERN +# SELECT regexp_extract('a', '(?Pa)', 1); # Wrong number of arguments statement error @@ -201,6 +287,32 @@ NULL statement error Expects group index between 0 and 1, but got 3 SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), ('12', 3)); +# Literal str with a column pattern +query T +SELECT regexp_extract('a1', column1, 1) FROM (VALUES ('(\d)'), ('([a-z])')); +---- +1 +a + +# Literal str and pattern with a column idx +query T +SELECT regexp_extract('a1b2', '([a-z])(\d)', column1) FROM (VALUES (0), (1), (2)); +---- +a1 +a +1 + +# An invalid pattern in a later row of a pattern column is an error +statement error INVALID_PARAMETER_VALUE.PATTERN.*is invalid: '\('\. +SELECT regexp_extract(column1, column2, 0) FROM (VALUES ('a', 'a'), ('b', '(')); + +# A NULL str row is not checked against its invalid pattern +query T +SELECT regexp_extract(column1, column2, 0) FROM (VALUES (CAST(NULL AS STRING), '('), ('b', 'b')); +---- +NULL +b + # Literal pattern with no match on any row and the default idx query T SELECT regexp_extract(column1, '\d+') FROM (VALUES ('abc'), ('xyz')); @@ -208,7 +320,8 @@ SELECT regexp_extract(column1, '\d+') FROM (VALUES ('abc'), ('xyz')); (empty) (empty) -# An invalid literal pattern is not compiled when every string is NULL +# An invalid literal pattern is not compiled when every string is NULL, as in +# Spark, so a literal pattern is not validated at planning time query T SELECT regexp_extract(column1, '(unclosed', 0) FROM (VALUES (CAST(NULL AS STRING)), (CAST(NULL AS STRING))); ---- @@ -221,12 +334,62 @@ SELECT regexp_extract(arrow_cast('1a 2b', 'LargeUtf8'), arrow_cast('(\d+)([a-z]) ---- a -# Integer types other than INT are accepted for idx +# Like Spark, str and regexp of other types are cast to strings +query T +SELECT regexp_extract(12345, '(\d)', 1); +---- +1 + +query T +SELECT regexp_extract(column1, '(\d)', 1) FROM (VALUES (12345), (67)); +---- +1 +6 + +query T +SELECT regexp_extract(X'6131', '(\d)', 1); +---- +1 + +query T +SELECT regexp_extract('a1', 1, 0); +---- +1 + +# Like Spark, idx of other numeric types and strings is cast to INT query T SELECT regexp_extract('1a 2b', '(\d+)([a-z])', arrow_cast(2, 'Int8')); ---- a +query T +SELECT regexp_extract('a1', '(\d)', '1'); +---- +1 + +query T +SELECT regexp_extract('a1', '(\d)', 1.0::DOUBLE); +---- +1 + +query T +SELECT regexp_extract('a1', '(\d)', 1.5::DOUBLE); +---- +1 + +# An idx that cannot be cast to INT is an error, as in Spark +# (Spark raises CAST_OVERFLOW and CAST_INVALID_INPUT) +statement error Can't cast value 4294967297 to type Int32 +SELECT regexp_extract('a1', '(\d)', 4294967297::BIGINT); + +statement error Cannot cast string 'x' to value of Int32 type +SELECT regexp_extract('a1', '(\d)', 'x'); + +# A str that cannot be cast to a string is a planning error, as in Spark +# (Spark raises DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE) +statement error Error during planning: regexp_extract expects a string str, got List +SELECT regexp_extract(array[1], 'a', 0); + # The result type is a string query T SELECT arrow_typeof(regexp_extract('a', '(a)')); From 874156790a0248cb60a600e57efad8d0e7e2d98c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:16:08 -0700 Subject: [PATCH 7/8] fix(spark): regexp_extract string types, pattern cache and error classes - Accept Utf8, LargeUtf8 and Utf8View for the subject and the pattern column, return Utf8, and error rather than panic on other types. - Size the output from the visible values, so a slice does not reserve its parent's buffer. - Cache literal patterns across batches on the UDF, keyed by pattern. - Classify rejected patterns with regex_syntax: only the syntax errors Java also raises are INVALID_PARAMETER_VALUE.PATTERN, and Java-only constructs such as \Q...\E, \Z, \h, (?>...) and (?d) are unsupported. - End both INVALID_PARAMETER_VALUE messages with SQLSTATE: 22023. - Assert idx cast errors under ANSI mode and record Spark's ANSI-off results, citing #88. --- Cargo.lock | 1 + Cargo.toml | 1 + crates/spark/Cargo.toml | 1 + .../src/function/string/regexp_extract.rs | 361 +++++++++++++++--- .../test_files/string/regexp_extract.slt | 69 +++- 5 files changed, 385 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 995e35c4..717edb0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1615,6 +1615,7 @@ dependencies = [ "percent-encoding", "rand 0.9.5", "regex", + "regex-syntax", "serde", "serde_json", "sha1", diff --git a/Cargo.toml b/Cargo.toml index afcce98f..a0e6da33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ log = "^0.4" num-traits = { version = "0.2" } rand = "0.9" regex = "1" +regex-syntax = "0.8" serde = "1" serde_json = "1" sha2 = "^0.11.0" diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index d63869c5..a267baa8 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -55,6 +55,7 @@ num-traits = { workspace = true } percent-encoding = "2.3.2" rand = { workspace = true } regex = { workspace = true } +regex-syntax = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha1 = "0.11" diff --git a/crates/spark/src/function/string/regexp_extract.rs b/crates/spark/src/function/string/regexp_extract.rs index 3fdc00b6..60ddd308 100644 --- a/crates/spark/src/function/string/regexp_extract.rs +++ b/crates/spark/src/function/string/regexp_extract.rs @@ -2,17 +2,21 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::sync::Arc; +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::sync::{Arc, Mutex}; use arrow::array::{ - Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, new_null_array, + Array, ArrayRef, AsArray, Int32Array, StringArray, StringArrayType, StringBuilder, + new_null_array, }; -use arrow::compute::can_cast_types; +use arrow::compute::{can_cast_types, cast}; use arrow::datatypes::DataType; use datafusion_common::cast::as_int32_array; +use datafusion_common::utils::offset_span_len; use datafusion_common::{ - DataFusionError, Result, ScalarValue, exec_datafusion_err, internal_datafusion_err, - internal_err, plan_err, + DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, + internal_datafusion_err, internal_err, plan_err, }; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -40,8 +44,12 @@ const NAME: &str = "regexp_extract"; and a NULL row is not checked, so an invalid pattern is only reported on a row it is applied to. \ As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \ Known differences from Spark (https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \ - `regex` engine, not Java's `java.util.regex`. Look-around, backreferences, and patterns that exceed the Rust \ - engine's compiled-size limit raise an \"unsupported regex pattern\" error, where Spark evaluates them. \ + `regex` engine, not Java's `java.util.regex`. Look-around, backreferences, quoting with `\\Q...\\E`, \ + `\\Z`, `\\h`, atomic groups `(?>...)`, the `(?d)` flag, and patterns that exceed the Rust engine's \ + compiled-size limit raise an \"unsupported regex pattern\" error, where Spark evaluates them. Only unclosed \ + or unopened groups and classes, invalid class ranges and repetitions, and unknown flags are reported as \ + `INVALID_PARAMETER_VALUE.PATTERN`; any other pattern the Rust engine rejects is reported as unsupported, \ + including a few that Java rejects too, such as `\\y`. \ Possessive quantifiers are read as nested greedy repetition (`a++` as `(a+)+`), so they can match \ where Java's would not. \ `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match only ASCII. \ @@ -75,9 +83,55 @@ NULL description = "Optional capture group index, cast to INT (numeric and string values are accepted). Defaults to 1; 0 is the whole match. NULL returns NULL." ) )] -#[derive(Debug, PartialEq, Eq, Hash)] +#[derive(Debug)] pub struct SparkRegexpExtract { signature: Signature, + /// Literal patterns compiled by earlier batches. One instance serves every + /// query in a session, so the cache is keyed by pattern. + patterns: PatternCache, +} + +// The cache does not affect the result, so two instances with the same +// signature are equal. +impl PartialEq for SparkRegexpExtract { + fn eq(&self, other: &Self) -> bool { + self.signature == other.signature + } +} + +impl Eq for SparkRegexpExtract {} + +impl Hash for SparkRegexpExtract { + fn hash(&self, state: &mut H) { + self.signature.hash(state); + } +} + +/// The literal patterns kept compiled. When it is full it is cleared, which +/// bounds its size if many different literal patterns are used. +const PATTERN_CACHE_SIZE: usize = 64; + +#[derive(Debug, Default)] +struct PatternCache(Mutex>); + +impl PatternCache { + fn get_or_compile(&self, pattern: &str) -> Result { + let lock = || { + self.0 + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + }; + if let Some(regex) = lock().get(pattern) { + return Ok(regex.clone()); + } + let regex = compile(pattern)?; + let mut patterns = lock(); + if patterns.len() >= PATTERN_CACHE_SIZE { + patterns.clear(); + } + patterns.insert(pattern.to_string(), regex.clone()); + Ok(regex) + } } impl Default for SparkRegexpExtract { @@ -90,6 +144,7 @@ impl SparkRegexpExtract { pub fn new() -> Self { Self { signature: Signature::user_defined(Volatility::Immutable), + patterns: PatternCache::default(), } } } @@ -151,12 +206,17 @@ impl ScalarUDFImpl for SparkRegexpExtract { // The usual call has a literal pattern and idx: keep those as scalars so // the pattern is compiled once and no per-row copies are allocated. - let pattern = match &args.args[1] { - ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? { + let pattern_column = match &args.args[1] { + ColumnarValue::Scalar(_) => None, + ColumnarValue::Array(array) => Some(to_utf8("regexp", array)?), + }; + let pattern = match (&args.args[1], &pattern_column) { + (_, Some(patterns)) => Pattern::Array(patterns.as_string::()), + (ColumnarValue::Scalar(scalar), None) => match scalar_str(scalar)? { Some(pattern) => Pattern::Scalar(pattern), None => return null_result(len, all_scalar), }, - ColumnarValue::Array(array) => Pattern::Array(array.as_string::()), + (ColumnarValue::Array(_), None) => unreachable!("converted above"), }; let index = match args.args.get(2) { None => Index::Scalar(1), @@ -175,8 +235,28 @@ impl ScalarUDFImpl for SparkRegexpExtract { Some(ColumnarValue::Array(array)) => Index::Array(as_int32_array(array)?), }; + // A caller that builds the physical expression directly, such as Comet, + // can pass any string type. Each result is a substring of its input, so + // the bytes of the visible input values bound the output. let strings = args.args[0].to_array(len)?; - let result = extract(strings.as_string::(), pattern, index)?; + let result = match strings.data_type() { + DataType::Utf8 => { + let strings = strings.as_string::(); + let capacity = offset_span_len(strings.offsets()); + extract(strings, capacity, pattern, index, &self.patterns)? + } + DataType::LargeUtf8 => { + let strings = strings.as_string::(); + let capacity = offset_span_len(strings.offsets()); + extract(strings, capacity, pattern, index, &self.patterns)? + } + DataType::Utf8View => { + let strings = strings.as_string_view(); + let capacity = strings.lengths().map(|len| len as usize).sum(); + extract(strings, capacity, pattern, index, &self.patterns)? + } + other => return exec_err!("{NAME} expects a string str, got {other}"), + }; if all_scalar { Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( &result, 0, @@ -199,6 +279,16 @@ enum Index<'a> { Array(&'a Int32Array), } +/// A pattern column as `Utf8`. Other string types are cast, and anything else +/// is an error. +fn to_utf8(name: &str, array: &ArrayRef) -> Result { + match array.data_type() { + DataType::Utf8 => Ok(Arc::clone(array)), + DataType::LargeUtf8 | DataType::Utf8View => Ok(cast(array, &DataType::Utf8)?), + other => exec_err!("{NAME} expects a string {name}, got {other}"), + } +} + fn scalar_str(scalar: &ScalarValue) -> Result> { match scalar { ScalarValue::Null => Ok(None), @@ -219,17 +309,20 @@ fn null_result(len: usize, all_scalar: bool) -> Result { }) } -fn extract(strings: &StringArray, pattern: Pattern, index: Index) -> Result { - // Each result is a substring of its input, so the input bytes bound the output. - let mut builder = - StringBuilder::with_capacity(strings.len(), strings.value_data().len()); - // A scalar pattern is compiled on the first non-NULL row and reused. A column - // pattern keeps only the last one compiled, since it rarely changes per row. - // The capture buffer is reused across rows instead of allocated per match. +fn extract<'a, S: StringArrayType<'a> + Copy>( + strings: S, + capacity: usize, + pattern: Pattern, + index: Index, + cache: &PatternCache, +) -> Result { + let mut builder = StringBuilder::with_capacity(strings.len(), capacity); + // A literal pattern comes from the cache on the first non-NULL row, so a NULL + // row is not checked. A column pattern keeps only the last one compiled, + // since it rarely changes per row. The capture buffer is reused across rows + // instead of allocated per match. let mut cached: Option<(&str, Regex, CaptureLocations)> = None; - let scalar_pattern = matches!(pattern, Pattern::Scalar(_)); - for row in 0..strings.len() { let row_pattern = match pattern { Pattern::Scalar(pattern) => Some(pattern), @@ -241,17 +334,21 @@ fn extract(strings: &StringArray, pattern: Pattern, index: Index) -> Result Some(idx), Index::Array(indexes) => (!indexes.is_null(row)).then(|| indexes.value(row)), }; - let (false, Some(pattern), Some(idx)) = (strings.is_null(row), row_pattern, idx) + let (false, Some(pattern_str), Some(idx)) = + (strings.is_null(row), row_pattern, idx) else { builder.append_null(); continue; }; let (_, regex, locations) = match &mut cached { - Some(entry) if scalar_pattern || entry.0 == pattern => entry, + Some(entry) if entry.0 == pattern_str => entry, _ => { - let regex = compile(pattern)?; + let regex = match pattern { + Pattern::Scalar(_) => cache.get_or_compile(pattern_str)?, + Pattern::Array(_) => compile(pattern_str)?, + }; let locations = regex.capture_locations(); - cached.insert((pattern, regex, locations)) + cached.insert((pattern_str, regex, locations)) } }; let subject = strings.value(row); @@ -276,45 +373,72 @@ fn check_group_index(regex: &Regex, idx: i32) -> Result { exec_datafusion_err!( "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of parameter(s) \ `idx` in `{NAME}` is invalid: Expects group index between 0 and \ - {group_count}, but got {idx}." + {group_count}, but got {idx}. SQLSTATE: 22023" ) }) } fn compile(pattern: &str) -> Result { Regex::new(pattern).map_err(|err| { - let pattern = to_sql_value(pattern); - match unsupported_reason(&err) { + let quoted = to_sql_value(pattern); + match unsupported_reason(pattern, &err) { Some(reason) => DataFusionError::NotImplemented(format!( - "Unsupported regex pattern {pattern} in `{NAME}`: {reason}. \ + "Unsupported regex pattern {quoted} in `{NAME}`: {reason}. \ See https://github.com/apache/datafusion-spark/issues/255" )), None => exec_datafusion_err!( "[INVALID_PARAMETER_VALUE.PATTERN] The value of parameter(s) `regexp` in \ - `{NAME}` is invalid: {pattern}." + `{NAME}` is invalid: {quoted}. SQLSTATE: 22023" ), } }) } -/// Returns why a pattern the Rust engine rejects may still be valid in Java, -/// for the errors the `regex` crate reports for features it does not support. -fn unsupported_reason(err: ®ex::Error) -> Option<&'static str> { - match err { - regex::Error::CompiledTooBig(_) => { - Some("the compiled pattern exceeds the Rust regex size limit") +/// Why a pattern that the Rust engine rejects may still be valid in Java, or +/// `None` if Java rejects it too. +/// +/// The pattern is parsed again with `regex_syntax`, whose error kinds are API, +/// rather than matching on the text of [`regex::Error`]. Only the kinds of +/// syntax error that Java also raises are reported as an invalid pattern. +/// Anything else, such as `\Q...\E`, `\Z`, `\h`, `(?>...)` or `(?d)`, is +/// reported as unsupported, which also covers a few patterns that Java rejects, +/// such as `\y`. +fn unsupported_reason(pattern: &str, err: ®ex::Error) -> Option { + use regex_syntax::ast::ErrorKind; + if let regex::Error::CompiledTooBig(_) = err { + return Some( + "the compiled pattern exceeds the Rust regex size limit".to_string(), + ); + } + let Err(ast_err) = regex_syntax::ast::parse::Parser::new().parse(pattern) else { + // The pattern parses, so a later stage of the Rust engine rejected it. + return Some(format!("the Rust regex engine rejects it ({err})")); + }; + match ast_err.kind() { + ErrorKind::GroupUnclosed + | ErrorKind::GroupUnopened + | ErrorKind::ClassUnclosed + | ErrorKind::ClassRangeInvalid + | ErrorKind::RepetitionMissing + | ErrorKind::RepetitionCountInvalid => None, + // Java also rejects an unknown flag, except `d` (UNIX_LINES) and `>` + // (an atomic group). + ErrorKind::FlagUnrecognized => { + let span = ast_err.span(); + let flag = &pattern[span.start.offset..span.end.offset]; + matches!(flag, "d" | ">").then(|| { + format!("the Rust regex engine does not support the flag '{flag}'") + }) } - regex::Error::Syntax(msg) - if msg.contains( - "look-around, including look-ahead and look-behind, is not supported", - ) => - { - Some("look-around is not supported by the Rust regex engine") + ErrorKind::UnsupportedLookAround => { + Some("look-around is not supported by the Rust regex engine".to_string()) } - regex::Error::Syntax(msg) if msg.contains("backreferences are not supported") => { - Some("backreferences are not supported by the Rust regex engine") + ErrorKind::UnsupportedBackreference => { + Some("backreferences are not supported by the Rust regex engine".to_string()) } - _ => None, + kind => Some(format!( + "the Rust regex engine does not support it ({kind})" + )), } } @@ -323,3 +447,150 @@ fn unsupported_reason(err: ®ex::Error) -> Option<&'static str> { fn to_sql_value(value: &str) -> String { format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")) } + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{LargeStringArray, StringViewArray}; + use arrow::datatypes::Field; + use datafusion_common::config::ConfigOptions; + + fn invoke( + udf: &SparkRegexpExtract, + args: Vec, + number_rows: usize, + ) -> Result { + let arg_fields = args + .iter() + .map(|arg| Arc::new(Field::new("a", arg.data_type(), true))) + .collect(); + udf.invoke_with_args(ScalarFunctionArgs { + args, + arg_fields, + number_rows, + return_field: Arc::new(Field::new(NAME, DataType::Utf8, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn pattern(pattern: &str) -> ColumnarValue { + ColumnarValue::Scalar(ScalarValue::Utf8(Some(pattern.to_string()))) + } + + fn strings(result: ColumnarValue) -> Vec> { + let array = result.into_array(1).unwrap(); + array + .as_string::() + .iter() + .map(|s| s.map(str::to_string)) + .collect() + } + + /// Ported from Comet's `large_utf8_subject_returns_utf8_array`: a caller + /// that builds the physical expression directly can pass any string type, + /// and the result is always `Utf8`. + #[test] + fn large_utf8_and_view_subjects_return_utf8() { + let udf = SparkRegexpExtract::new(); + let subjects: [ArrayRef; 2] = [ + Arc::new(LargeStringArray::from(vec![Some("a1"), None, Some("b22")])), + Arc::new(StringViewArray::from(vec![Some("a1"), None, Some("b22")])), + ]; + for subject in subjects { + let result = invoke( + &udf, + vec![ColumnarValue::Array(subject), pattern(r"(\d+)")], + 3, + ) + .unwrap(); + assert_eq!(result.data_type(), DataType::Utf8); + assert_eq!( + strings(result), + vec![Some("1".to_string()), None, Some("22".to_string())] + ); + } + } + + #[test] + fn large_utf8_pattern_column() { + let udf = SparkRegexpExtract::new(); + let subject: ArrayRef = Arc::new(StringArray::from(vec!["a1", "b2"])); + let patterns: ArrayRef = + Arc::new(LargeStringArray::from(vec![r"(\d)", "([a-z])"])); + let result = invoke( + &udf, + vec![ + ColumnarValue::Array(subject), + ColumnarValue::Array(patterns), + ], + 2, + ) + .unwrap(); + assert_eq!( + strings(result), + vec![Some("1".to_string()), Some("b".to_string())] + ); + } + + #[test] + fn non_string_subject_is_an_error() { + let udf = SparkRegexpExtract::new(); + let subject: ArrayRef = Arc::new(Int32Array::from(vec![1])); + let error = invoke(&udf, vec![ColumnarValue::Array(subject), pattern("(1)")], 1) + .unwrap_err(); + assert!( + error + .to_string() + .contains("expects a string str, got Int32"), + "{error}" + ); + } + + /// The output is sized from the visible values, not the whole buffer that + /// a slice shares with its parent. + #[test] + fn sliced_subject_allocates_for_the_slice() { + let udf = SparkRegexpExtract::new(); + let padding = "x".repeat(65536); + let parent = StringArray::from(vec![padding.as_str(), "a1", padding.as_str()]); + for (offset, len, expected) in + [(1, 0, vec![]), (1, 1, vec![Some("1".to_string())])] + { + let subject: ArrayRef = Arc::new(parent.slice(offset, len)); + let result = invoke( + &udf, + vec![ColumnarValue::Array(subject), pattern(r"(\d)")], + len, + ) + .unwrap() + .into_array(len) + .unwrap(); + let result = result.as_string::(); + assert_eq!( + result + .iter() + .map(|s| s.map(str::to_string)) + .collect::>(), + expected + ); + assert!( + result.values().capacity() < 1024, + "allocated {} bytes", + result.values().capacity() + ); + } + } + + #[test] + fn literal_patterns_are_cached_across_batches() { + let udf = SparkRegexpExtract::new(); + for subject in ["a1", "b2"] { + let subject = + ColumnarValue::Array(Arc::new(StringArray::from(vec![subject]))); + invoke(&udf, vec![subject, pattern(r"(\d)")], 1).unwrap(); + } + let patterns = udf.patterns.0.lock().unwrap(); + assert_eq!(patterns.len(), 1); + assert!(patterns.contains_key(r"(\d)")); + } +} diff --git a/crates/sqllogictest/test_files/string/regexp_extract.slt b/crates/sqllogictest/test_files/string/regexp_extract.slt index 430931c8..2a612b94 100644 --- a/crates/sqllogictest/test_files/string/regexp_extract.slt +++ b/crates/sqllogictest/test_files/string/regexp_extract.slt @@ -127,7 +127,7 @@ SELECT regexp_extract('a', NULL, 99); NULL # Group index out of range -statement error INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX.*Expects group index between 0 and 0, but got 1 +statement error ^.*\[INVALID_PARAMETER_VALUE\.REGEX_GROUP_INDEX\] The value of parameter\(s\) `idx` in `regexp_extract` is invalid: Expects group index between 0 and 0, but got 1\. SQLSTATE: 22023$ SELECT regexp_extract('1a 2b 14m', '\d+'); statement error Expects group index between 0 and 0, but got 1 @@ -146,7 +146,7 @@ statement error Expects group index between 0 and 2, but got -1 SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1); # Invalid pattern -statement error INVALID_PARAMETER_VALUE.PATTERN +statement error ^.*\[INVALID_PARAMETER_VALUE\.PATTERN\] The value of parameter\(s\) `regexp` in `regexp_extract` is invalid: '\(\?l\)'\. SQLSTATE: 22023$ SELECT regexp_extract('1a 2b 14m', '(?l)'); statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed' @@ -165,6 +165,12 @@ SELECT regexp_extract('abc', '', 0); ---- (empty) +# A zero-width group, and a group that ends after a multi-byte character +query TT +SELECT regexp_extract('éx', '()', 1), regexp_extract('éx', '(é)', 1); +---- +(empty) é + # Known differences between the Rust regex engine and java.util.regex # https://github.com/apache/datafusion-spark/issues/255 @@ -189,6 +195,30 @@ SELECT regexp_extract('aa', '(a)\1', 0); # ---- # aa +# Java constructs that the Rust regex engine does not have are unsupported, not +# invalid: quoting, \Z, \h, atomic groups and the d flag +statement error Unsupported regex pattern '\\\\Q\.\\\\E' in `regexp_extract` +SELECT regexp_extract('a.b', '\Q.\E', 0); + +statement error Unsupported regex pattern .* in `regexp_extract` +SELECT regexp_extract('ab', 'b\Z', 0); + +statement error Unsupported regex pattern .* in `regexp_extract` +SELECT regexp_extract('a b', '\h', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: the Rust regex engine does not support the flag '>' +SELECT regexp_extract('aab', '(?>a+)b', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: the Rust regex engine does not support the flag 'd' +SELECT regexp_extract('ab', '(?d)b', 0); + +# Spark 4.2.0 results for those patterns, in both ANSI modes +# https://github.com/apache/datafusion-spark/issues/255 +# query TTTTT +# SELECT regexp_extract('a.b', '\Q.\E', 0), regexp_extract('ab', 'b\Z', 0), concat('[', regexp_extract('a b', '\h', 0), ']'), regexp_extract('aab', '(?>a+)b', 0), regexp_extract('ab', '(?d)b', 0); +# ---- +# . b [ ] aab b + # Spark 4.2.0 treats a++ as a possessive quantifier that does not give back, # so a++a cannot match; the Rust regex engine reads a++ as (a+)+ and returns 'aaa' # https://github.com/apache/datafusion-spark/issues/255 @@ -377,14 +407,47 @@ SELECT regexp_extract('a1', '(\d)', 1.5::DOUBLE); ---- 1 -# An idx that cannot be cast to INT is an error, as in Spark +# A string idx column is cast to INT +query T +SELECT regexp_extract('a1', '([a-z])(\d)', column1) FROM (VALUES ('0'), ('1'), ('2'), (CAST(NULL AS STRING))); +---- +a1 +a +1 +NULL + +# With ANSI mode on, an idx that cannot be cast to INT is an error, as in Spark # (Spark raises CAST_OVERFLOW and CAST_INVALID_INPUT) +statement ok +set datafusion.execution.enable_ansi_mode = true; + statement error Can't cast value 4294967297 to type Int32 SELECT regexp_extract('a1', '(\d)', 4294967297::BIGINT); statement error Cannot cast string 'x' to value of Int32 type SELECT regexp_extract('a1', '(\d)', 'x'); +statement ok +set datafusion.execution.enable_ansi_mode = false; + +# With ANSI mode off, Spark 4.2.0 wraps an out-of-range BIGINT (to 1 here), returns +# NULL for a malformed string, and casts NaN to 0. DataFusion's cast errors instead. +# https://github.com/apache/datafusion-spark/issues/88 +# query T +# SELECT regexp_extract('a1', '(\d)', 4294967297::BIGINT); +# ---- +# 1 +# +# query T +# SELECT regexp_extract('a1', '(\d)', 'x'); +# ---- +# NULL +# +# query T +# SELECT regexp_extract('a1', '(\d)', 'NaN'::DOUBLE); +# ---- +# 1 + # A str that cannot be cast to a string is a planning error, as in Spark # (Spark raises DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE) statement error Error during planning: regexp_extract expects a string str, got List From 9a984b9c24312b8d897b35a17bd2a917c4fbe111 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:58:11 -0700 Subject: [PATCH 8/8] fix(spark): reject regexp_extract patterns Java reads differently Walk the parsed AST of a pattern before compiling it, so syntax that both engines accept but read differently no longer returns a different answer than Spark with no error. The U flag, POSIX classes in brackets, the class operators -- and ~~, \< and \>, and Java's ASCII-only POSIX names such as \p{Alpha} are reported as unsupported. Syntax that Java rejects, the R flag, \b{...} word boundaries, (?P...) and group names Java does not allow, is INVALID_PARAMETER_VALUE.PATTERN, as in Spark. Count compiles in the pattern cache under cfg(test), so the cache test fails when a later batch compiles the pattern again instead of reading it. --- .../src/function/string/regexp_extract.rs | 270 +++++++++++++++--- .../test_files/string/regexp_extract.slt | 102 ++++++- 2 files changed, 326 insertions(+), 46 deletions(-) diff --git a/crates/spark/src/function/string/regexp_extract.rs b/crates/spark/src/function/string/regexp_extract.rs index 60ddd308..14bb8dbe 100644 --- a/crates/spark/src/function/string/regexp_extract.rs +++ b/crates/spark/src/function/string/regexp_extract.rs @@ -24,6 +24,10 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; use regex::{CaptureLocations, Regex}; +use regex_syntax::ast::{ + AssertionKind, Ast, ClassSetBinaryOp, ClassSetBinaryOpKind, ClassSetItem, + ClassUnicode, ClassUnicodeKind, Flag, Flags, FlagsItemKind, GroupKind, Visitor, +}; use crate::function::error_utils::invalid_arg_count_exec_err; @@ -50,13 +54,20 @@ const NAME: &str = "regexp_extract"; or unopened groups and classes, invalid class ranges and repetitions, and unknown flags are reported as \ `INVALID_PARAMETER_VALUE.PATTERN`; any other pattern the Rust engine rejects is reported as unsupported, \ including a few that Java rejects too, such as `\\y`. \ + Rust syntax that Java rejects is also `INVALID_PARAMETER_VALUE.PATTERN`: the `R` flag, `\\b{start}` and \ + the other `\\b{...}` word boundaries, `(?P...)`, and group names that are not an ASCII letter \ + followed by ASCII letters and digits. Syntax that both engines accept but read differently is reported as \ + unsupported rather than evaluated to a different answer: the `U` flag (Java's UNICODE_CHARACTER_CLASS, \ + which swaps greediness in Rust), `[:alpha:]` and the other POSIX classes in brackets, the class operators \ + `--` and `~~`, `\\<` and `\\>`, and the names of Java's ASCII-only POSIX classes, such as `\\p{Alpha}` \ + and `\\p{Lower}`. Other Unicode property names that Java does not know, such as `\\p{Letter}` or \ + `\\p{Latin}`, are accepted, where Java rejects them. \ Possessive quantifiers are read as nested greedy repetition (`a++` as `(a+)+`), so they can match \ where Java's would not. \ `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match only ASCII. \ `.` matches `\\r`, where Java's does not. `$` matches only at the very end of the input, where Java's also \ matches before a final line terminator. `(?i)` folds case using Unicode rules (so `k` matches the Kelvin \ - sign), where Java folds only ASCII by default. Rust's `(?P...)` named groups are accepted, where Java \ - rejects them as an invalid pattern.", + sign), where Java folds only ASCII by default.", syntax_example = "regexp_extract(str, regexp[, idx])", sql_example = r#"```sql > SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)'); @@ -112,18 +123,27 @@ impl Hash for SparkRegexpExtract { const PATTERN_CACHE_SIZE: usize = 64; #[derive(Debug, Default)] -struct PatternCache(Mutex>); +struct PatternCache { + patterns: Mutex>, + /// How many patterns were compiled, so a test can tell that a later batch + /// read the cache rather than compiling and inserting the pattern again. + #[cfg(test)] + compiles: std::sync::atomic::AtomicUsize, +} impl PatternCache { fn get_or_compile(&self, pattern: &str) -> Result { let lock = || { - self.0 + self.patterns .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) }; if let Some(regex) = lock().get(pattern) { return Ok(regex.clone()); } + #[cfg(test)] + self.compiles + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); let regex = compile(pattern)?; let mut patterns = lock(); if patterns.len() >= PATTERN_CACHE_SIZE { @@ -379,69 +399,225 @@ fn check_group_index(regex: &Regex, idx: i32) -> Result { } fn compile(pattern: &str) -> Result { - Regex::new(pattern).map_err(|err| { - let quoted = to_sql_value(pattern); - match unsupported_reason(pattern, &err) { - Some(reason) => DataFusionError::NotImplemented(format!( - "Unsupported regex pattern {quoted} in `{NAME}`: {reason}. \ - See https://github.com/apache/datafusion-spark/issues/255" - )), - None => exec_datafusion_err!( + let quoted = to_sql_value(pattern); + let unsupported = |reason: String| { + DataFusionError::NotImplemented(format!( + "Unsupported regex pattern {quoted} in `{NAME}`: {reason}. \ + See https://github.com/apache/datafusion-spark/issues/255" + )) + }; + match java_difference(pattern) { + None => {} + Some(JavaDifference::Invalid) => { + return Err(exec_datafusion_err!( "[INVALID_PARAMETER_VALUE.PATTERN] The value of parameter(s) `regexp` in \ `{NAME}` is invalid: {quoted}. SQLSTATE: 22023" - ), + )); } + Some(JavaDifference::Unsupported(reason)) => return Err(unsupported(reason)), + } + // The pattern parsed, so a later stage of the Rust engine rejected it. + Regex::new(pattern).map_err(|err| match err { + regex::Error::CompiledTooBig(_) => unsupported( + "the compiled pattern exceeds the Rust regex size limit".to_string(), + ), + err => unsupported(format!("the Rust regex engine rejects it ({err})")), }) } -/// Why a pattern that the Rust engine rejects may still be valid in Java, or -/// `None` if Java rejects it too. +/// How `java.util.regex`, which Spark uses, would treat a pattern differently +/// from the Rust regex engine. +#[derive(Debug)] +enum JavaDifference { + /// Java rejects the pattern, so it is reported as an invalid pattern. + Invalid, + /// Java accepts the pattern but the Rust engine rejects it, or reads it + /// differently, so it is reported as unsupported rather than evaluated to + /// a different answer than Spark's. + Unsupported(String), +} + +/// Checks a pattern with `regex_syntax`, whose AST and error kinds are API, +/// rather than matching on the text of [`regex::Error`]. /// -/// The pattern is parsed again with `regex_syntax`, whose error kinds are API, -/// rather than matching on the text of [`regex::Error`]. Only the kinds of -/// syntax error that Java also raises are reported as an invalid pattern. -/// Anything else, such as `\Q...\E`, `\Z`, `\h`, `(?>...)` or `(?d)`, is -/// reported as unsupported, which also covers a few patterns that Java rejects, -/// such as `\y`. -fn unsupported_reason(pattern: &str, err: ®ex::Error) -> Option { +/// A pattern that does not parse is invalid only for the kinds of syntax error +/// that Java also raises. Anything else, such as `\Q...\E`, `\Z`, `\h`, +/// `(?>...)` or `(?d)`, is unsupported, which also covers a few patterns that +/// Java rejects, such as `\y`. A pattern that parses is walked for the +/// constructs that the two engines read differently. +fn java_difference(pattern: &str) -> Option { use regex_syntax::ast::ErrorKind; - if let regex::Error::CompiledTooBig(_) = err { - return Some( - "the compiled pattern exceeds the Rust regex size limit".to_string(), - ); - } - let Err(ast_err) = regex_syntax::ast::parse::Parser::new().parse(pattern) else { - // The pattern parses, so a later stage of the Rust engine rejected it. - return Some(format!("the Rust regex engine rejects it ({err})")); + let ast_err = match regex_syntax::ast::parse::Parser::new().parse(pattern) { + Ok(ast) => return regex_syntax::ast::visit(&ast, JavaDifferences).err(), + Err(err) => err, }; + let unsupported = |reason: String| Some(JavaDifference::Unsupported(reason)); match ast_err.kind() { ErrorKind::GroupUnclosed | ErrorKind::GroupUnopened | ErrorKind::ClassUnclosed | ErrorKind::ClassRangeInvalid | ErrorKind::RepetitionMissing - | ErrorKind::RepetitionCountInvalid => None, + | ErrorKind::RepetitionCountInvalid => Some(JavaDifference::Invalid), // Java also rejects an unknown flag, except `d` (UNIX_LINES) and `>` // (an atomic group). ErrorKind::FlagUnrecognized => { let span = ast_err.span(); let flag = &pattern[span.start.offset..span.end.offset]; - matches!(flag, "d" | ">").then(|| { - format!("the Rust regex engine does not support the flag '{flag}'") - }) - } - ErrorKind::UnsupportedLookAround => { - Some("look-around is not supported by the Rust regex engine".to_string()) - } - ErrorKind::UnsupportedBackreference => { - Some("backreferences are not supported by the Rust regex engine".to_string()) + if matches!(flag, "d" | ">") { + unsupported(format!( + "the Rust regex engine does not support the flag '{flag}'" + )) + } else { + Some(JavaDifference::Invalid) + } } - kind => Some(format!( + ErrorKind::UnsupportedLookAround => unsupported( + "look-around is not supported by the Rust regex engine".to_string(), + ), + ErrorKind::UnsupportedBackreference => unsupported( + "backreferences are not supported by the Rust regex engine".to_string(), + ), + kind => unsupported(format!( "the Rust regex engine does not support it ({kind})" )), } } +/// Java's POSIX character classes, which match ASCII only (unless the `U` flag +/// is set, which is unsupported). The Rust engine reads the same names as +/// Unicode properties, so `\p{Alpha}` matches `é` there but not in Java. +const JAVA_POSIX_CLASSES: [&str; 13] = [ + "Lower", "Upper", "ASCII", "Alpha", "Digit", "Alnum", "Punct", "Graph", "Print", + "Blank", "Cntrl", "XDigit", "Space", +]; + +/// Finds the constructs of a parsed pattern that Java rejects or reads +/// differently. Each is checked by its AST node, so an escaped or quoted +/// look-alike, such as `\[\[:alpha:\]\]`, is not reported. +struct JavaDifferences; + +impl JavaDifferences { + fn check_flags(flags: &Flags) -> Result<(), JavaDifference> { + // Java has no R flag, even negated. + if flags + .items + .iter() + .any(|item| item.kind == FlagsItemKind::Flag(Flag::CRLF)) + { + return Err(JavaDifference::Invalid); + } + // `(?-U)` is the default in both engines, so only setting it differs. + if flags.flag_state(Flag::SwapGreed) == Some(true) { + return Err(JavaDifference::Unsupported( + "Java's flag 'U' is UNICODE_CHARACTER_CLASS, where the Rust regex \ + engine's swaps greediness" + .to_string(), + )); + } + Ok(()) + } + + fn check_unicode_class(class: &ClassUnicode) -> Result<(), JavaDifference> { + match &class.kind { + ClassUnicodeKind::Named(name) + if JAVA_POSIX_CLASSES.contains(&name.as_str()) => + { + let p = if class.negated { 'P' } else { 'p' }; + Err(JavaDifference::Unsupported(format!( + "Java's \\{p}{{{name}}} is an ASCII-only POSIX class, where the Rust \ + regex engine's is a Unicode property" + ))) + } + _ => Ok(()), + } + } +} + +impl Visitor for JavaDifferences { + type Output = (); + type Err = JavaDifference; + + fn finish(self) -> Result<(), JavaDifference> { + Ok(()) + } + + fn visit_pre(&mut self, ast: &Ast) -> Result<(), JavaDifference> { + match ast { + Ast::Flags(set) => Self::check_flags(&set.flags), + Ast::Group(group) => match &group.kind { + GroupKind::NonCapturing(flags) => Self::check_flags(flags), + // Java rejects `(?P...)`, and a name that is not an ASCII + // letter followed by ASCII letters and digits. + GroupKind::CaptureName { starts_with_p, name } => { + let mut chars = name.name.chars(); + let valid = !starts_with_p + && chars.next().is_some_and(|c| c.is_ascii_alphabetic()) + && chars.all(|c| c.is_ascii_alphanumeric()); + if valid { + Ok(()) + } else { + Err(JavaDifference::Invalid) + } + } + GroupKind::CaptureIndex(_) => Ok(()), + }, + Ast::Assertion(assertion) => match assertion.kind { + // Java rejects `\b{start}`, `\b{end}`, `\b{start-half}` and + // `\b{end-half}`. + AssertionKind::WordBoundaryStart + | AssertionKind::WordBoundaryEnd + | AssertionKind::WordBoundaryStartHalf + | AssertionKind::WordBoundaryEndHalf => Err(JavaDifference::Invalid), + AssertionKind::WordBoundaryStartAngle + | AssertionKind::WordBoundaryEndAngle => Err(JavaDifference::Unsupported( + "Java reads \\< and \\> as literal characters, where the Rust regex \ + engine reads them as word boundaries" + .to_string(), + )), + AssertionKind::StartLine + | AssertionKind::EndLine + | AssertionKind::StartText + | AssertionKind::EndText + | AssertionKind::WordBoundary + | AssertionKind::NotWordBoundary => Ok(()), + }, + Ast::ClassUnicode(class) => Self::check_unicode_class(class), + _ => Ok(()), + } + } + + fn visit_class_set_item_pre( + &mut self, + item: &ClassSetItem, + ) -> Result<(), JavaDifference> { + match item { + ClassSetItem::Ascii(_) => Err(JavaDifference::Unsupported( + "Java reads [:name:] inside a class as a nested class of its characters, \ + where the Rust regex engine reads it as a POSIX class" + .to_string(), + )), + ClassSetItem::Unicode(class) => Self::check_unicode_class(class), + _ => Ok(()), + } + } + + fn visit_class_set_binary_op_pre( + &mut self, + op: &ClassSetBinaryOp, + ) -> Result<(), JavaDifference> { + let operator = match op.kind { + // `&&` is intersection in both engines. + ClassSetBinaryOpKind::Intersection => return Ok(()), + ClassSetBinaryOpKind::Difference => "--", + ClassSetBinaryOpKind::SymmetricDifference => "~~", + }; + Err(JavaDifference::Unsupported(format!( + "Java has no class operator '{operator}', which it reads as literal characters" + ))) + } +} + /// Quotes a string the way Spark's `toSQLValue` renders a string literal in an /// error message: wrapped in single quotes, with `\` and `'` escaped. fn to_sql_value(value: &str) -> String { @@ -589,8 +765,14 @@ mod tests { ColumnarValue::Array(Arc::new(StringArray::from(vec![subject]))); invoke(&udf, vec![subject, pattern(r"(\d)")], 1).unwrap(); } - let patterns = udf.patterns.0.lock().unwrap(); - assert_eq!(patterns.len(), 1); - assert!(patterns.contains_key(r"(\d)")); + // Counting compiles, not cache entries, since a cache that is never + // read would also end up with one entry for the pattern. + assert_eq!( + udf.patterns + .compiles + .load(std::sync::atomic::Ordering::Relaxed), + 1 + ); + assert!(udf.patterns.patterns.lock().unwrap().contains_key(r"(\d)")); } } diff --git a/crates/sqllogictest/test_files/string/regexp_extract.slt b/crates/sqllogictest/test_files/string/regexp_extract.slt index 2a612b94..7e618d33 100644 --- a/crates/sqllogictest/test_files/string/regexp_extract.slt +++ b/crates/sqllogictest/test_files/string/regexp_extract.slt @@ -260,10 +260,108 @@ SELECT regexp_extract('a', '\w{1000}\w{1000}\w{1000}', 0); # ---- # (empty) -# Spark 4.2.0 rejects Python-style named groups; DataFusion returns 'a' +# Patterns that the Rust regex engine accepts but Java rejects are invalid, as +# in Spark 4.2.0: the R flag, \b{...} word boundaries, Python-style named groups +# and group names that are not an ASCII letter followed by letters and digits +statement error ^.*\[INVALID_PARAMETER_VALUE\.PATTERN\] The value of parameter\(s\) `regexp` in `regexp_extract` is invalid: '\(\?R\)a\$'\. SQLSTATE: 22023$ +SELECT regexp_extract('a', '(?R)a$', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('a', '(?-R)a', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('abc', '\b{start}abc', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('abc', 'abc\b{end}', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('abc', '\b{start-half}abc', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('abc', 'abc\b{end-half}', 0); + +statement error INVALID_PARAMETER_VALUE.PATTERN.*is invalid: '\(\?Pa\)'\. +SELECT regexp_extract('a', '(?Pa)', 1); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('x', '(?x)', 1); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('x', '(?<_a>x)', 1); + +statement error INVALID_PARAMETER_VALUE.PATTERN +SELECT regexp_extract('x', '(?x)', 1); + +query T +SELECT regexp_extract('x', '(?x)', 1); +---- +x + +# Patterns that both engines accept but read differently are unsupported, so +# they do not silently return a different answer than Spark. Java's U flag is +# UNICODE_CHARACTER_CLASS, where the Rust engine's swaps greediness. +statement error Unsupported regex pattern '\(\?U\)a\+' in `regexp_extract`: Java's flag 'U' +SELECT regexp_extract('aaa', '(?U)a+', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java's flag 'U' +SELECT regexp_extract('aaa', '(?U:a+)', 0); + +# Java reads [[:alpha:]] as a nested class of the characters ':alph' +statement error Unsupported regex pattern .* in `regexp_extract`: Java reads \[:name:\] +SELECT regexp_extract('x', '[[:alpha:]]', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java reads \[:name:\] +SELECT regexp_extract('ab', '[[:^alpha:]]', 0); + +# Java has no class difference operators +statement error Unsupported regex pattern .* in `regexp_extract`: Java has no class operator '--' +SELECT regexp_extract('b', '[a-c--b]', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java has no class operator '~~' +SELECT regexp_extract('~', '[a~~b]', 0); + +# Java reads \< and \> as the literal characters +statement error Unsupported regex pattern .* in `regexp_extract`: Java reads \\< and \\> as literal characters +SELECT regexp_extract('a as literal characters +SELECT regexp_extract('a>', 'a\>', 0); + +# Java's POSIX classes match ASCII only, where the Rust engine reads the same +# names as Unicode properties +statement error Unsupported regex pattern '\\\\p\{Alpha\}' in `regexp_extract`: Java's \\p\{Alpha\} is an ASCII-only POSIX class +SELECT regexp_extract('é', '\p{Alpha}', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java's \\P\{Lower\} is an ASCII-only POSIX class +SELECT regexp_extract('é', '\P{Lower}', 0); + +statement error Unsupported regex pattern .* in `regexp_extract`: Java's \\p\{Space\} is an ASCII-only POSIX class +SELECT regexp_extract('a', '[a\p{Space}]', 0); + +# A pattern column is checked the same way +statement error Unsupported regex pattern .* in `regexp_extract`: Java's flag 'U' +SELECT regexp_extract(column1, column2, 0) FROM (VALUES ('aaa', 'a+'), ('aaa', '(?U)a+')); + +# Spark 4.2.0 results for those patterns, in both ANSI modes +# https://github.com/apache/datafusion-spark/issues/255 +# query TTTTTTTTTTT +# SELECT regexp_extract('aaa', '(?U)a+', 0), concat('[', regexp_extract('x', '[[:alpha:]]', 0), ']'), regexp_extract(':', '[[:alpha:]]', 0), regexp_extract('ab', '[[:^alpha:]]', 0), regexp_extract('b', '[a-c--b]', 0), regexp_extract('~', '[a~~b]', 0), regexp_extract('a', 'a\>', 0), concat('[', regexp_extract('é', '\p{Alpha}', 0), ']'), regexp_extract('é', '\P{Lower}', 0), regexp_extract('1', '\p{Digit}', 0); +# ---- +# aaa [] : a b ~ [] é 1 + +# Constructs that mean the same in both engines are accepted: the default (?-U), +# class intersection and Unicode properties that are not POSIX names +query TTT +SELECT regexp_extract('aaa', '(?-U)a+', 0), regexp_extract('b', '[a-c&&b]', 0), regexp_extract('é', '\p{L}', 0); +---- +aaa b é + +# Spark 4.2.0 rejects Unicode property names that Java does not know, such as +# \p{Letter}, \p{Latin} and \p{alpha}; DataFusion returns 'é' # https://github.com/apache/datafusion-spark/issues/255 # statement error INVALID_PARAMETER_VALUE.PATTERN -# SELECT regexp_extract('a', '(?Pa)', 1); +# SELECT regexp_extract('é', '\p{Letter}', 0); # Wrong number of arguments statement error