diff --git a/docs/source/contributor-guide/expression-audits/agg_funcs.md b/docs/source/contributor-guide/expression-audits/agg_funcs.md index c3157c0ba6d..d3de216a288 100644 --- a/docs/source/contributor-guide/expression-audits/agg_funcs.md +++ b/docs/source/contributor-guide/expression-audits/agg_funcs.md @@ -72,6 +72,15 @@ - Performance (tuned 2026-09-09, PR [#5803](https://github.com/apache/datafusion-comet/pull/5803)): as for [collect_list](#collect-list), grouped `collect_set` no longer goes through `GroupsAccumulatorAdapter`. `CollectSetGroupsAccumulator` encodes each batch once for all of its groups and deduplicates against an open-addressed index over a shared arena, replacing one hash table plus one owned `Row` per distinct value per group. 60-93% faster ([#5797](https://github.com/apache/datafusion-comet/issues/5797)). Benchmark: `benches/collect.rs`. - `CometCollectSet` reports `Incompatible` for float and double input when `spark.comet.exec.strictFloatingPoint=true`, because the native distinct comparison treats `NaN == NaN` and collapses repeated `NaN`s into a single element while Spark keeps each one. The native path for floating-point input is then opt-in via `spark.comet.expression.CollectSet.allowIncompatible=true`. All other input types are `Compatible`. +## kurtosis + +- Spark 3.4.3 (audited 2026-07-03): `Kurtosis(child, nullOnDivideByZero)` extends `CentralMomentAgg` with `momentOrder = 4`. Excess kurtosis (Fisher), formula `n * m4 / (m2 * m2) - 3.0`; empty group → `NULL`; `m2 == 0` → `NULL` when `nullOnDivideByZero=true` (default when `spark.sql.legacy.statisticalAggregate=false`) else `NaN`. Any numeric input is cast to `Double` by `ImplicitCastInputTypes`. +- Spark 3.5.8 (audited 2026-07-03): identical to 3.4.3. +- Spark 4.0.1 (audited 2026-07-03): identical to 3.4.3. No collation involvement. +- Spark 4.1.1 (audited 2026-07-03): identical to 3.4.3. +- `CometKurtosis` maps the aggregate to a Comet-owned `Kurtosis` UDAF whose intermediate state (`[n, avg, m2, m3, m4]` Float64) mirrors Spark's `CentralMomentAgg` buffer for `momentOrder = 4`, so Partial output produced by either engine has the same wire format. The Rust update/merge kernels are a direct port of Spark's `updateExpressionsDef` and `mergeExpressions`. `supportsSparkPartialToNativeFinal` and `supportsNativePartialToSparkFinal` are left at the default `false`, matching the conservative policy the other `CentralMomentAgg` serdes (`Variance`, `Stddev`) already use in the same file. +- Window use (`kurtosis(x) OVER (...)`) currently falls back to Spark: the window path doesn't wire the Comet aggregate for kurtosis today. + ## listagg - Spark 3.4.3 (audited 2026-07-03): does not exist. `ListAgg` was added in Spark 4.0. diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 58ba641a2f8..d2285eb9864 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -108,7 +108,7 @@ The tables below list every Spark built-in expression with its current status. | `grouping_id` | ✅ | — | Grouping indicator for ROLLUP/CUBE/GROUPING SETS | | `hll_sketch_agg` | ✅ | Native | Spark 4.0+ only; falls back by default, the native path is opt-in via allowIncompatible ([details](compatibility/expressions/aggregate.md)) | | `hll_union_agg` | ✅ | Native | Spark 4.0+ only; falls back by default, the native path is opt-in via allowIncompatible ([details](compatibility/expressions/aggregate.md)) | -| `kurtosis` | 🔜 | — | Not yet implemented natively | +| `kurtosis` | ✅ | Native | Excess kurtosis (Fisher definition). | | `last` | ✅ | Native | | | `last_value` | ✅ | Native | | | `listagg` | ✅ | Native | Spark 4.0+. `StringType` input with a literal delimiter; `WITHIN GROUP (ORDER BY ...)` and `BinaryType` inputs fall back to Spark. Without `WITHIN GROUP`, concatenation order is the (non-deterministic) group arrival order, matching Spark. | diff --git a/native/core/src/execution/planner/aggregate.rs b/native/core/src/execution/planner/aggregate.rs index 1c03dd0d7a6..9372d4d2060 100644 --- a/native/core/src/execution/planner/aggregate.rs +++ b/native/core/src/execution/planner/aggregate.rs @@ -52,8 +52,8 @@ use datafusion_comet_proto::{ }; use datafusion_comet_spark_expr::{ ApproxPercentile, Avg, AvgDecimal, BloomFilterAgg, Cast, CometCollectList, CometCollectSet, - Correlation, Covariance, EvalMode, HllPlusPlus, HllSketchAgg, HllUnionAgg, MaxMinBy, Mode, - Regr, RegrType, SparkBloomFilterVersion, SparkCastOptions, SparkListAgg, SparkMinMax, + Correlation, Covariance, EvalMode, HllPlusPlus, HllSketchAgg, HllUnionAgg, Kurtosis, MaxMinBy, + Mode, Regr, RegrType, SparkBloomFilterVersion, SparkCastOptions, SparkListAgg, SparkMinMax, SparkPercentile, SparkPivotFirst, SparkStructMinMax, Stddev, SumDecimal, SumInteger, Variance, ZeroOrder, }; @@ -622,6 +622,17 @@ impl PhysicalPlanner { ))), } } + AggExprStruct::Kurtosis(expr) => { + let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&schema))?; + let func = AggregateUDF::new_from_impl(Kurtosis::new( + "kurtosis", + expr.null_on_divide_by_zero, + from_protobuf_eval_mode(expr.eval_mode)?, + spark_expr.expr_id, + Arc::clone(&self.query_context_registry), + )); + Self::create_aggr_func_expr("kurtosis", schema, vec![child], func) + } AggExprStruct::Stddev(expr) => { let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&schema))?; let datatype = to_arrow_datatype(expr.datatype.as_ref().unwrap()); diff --git a/native/proto/src/proto/expr.proto b/native/proto/src/proto/expr.proto index 4ff0d99ef4f..c3d74f82f98 100644 --- a/native/proto/src/proto/expr.proto +++ b/native/proto/src/proto/expr.proto @@ -172,6 +172,7 @@ message AggExpr { HllSketchAgg hllSketchAgg = 27; HllUnionAgg hllUnionAgg = 28; PivotFirst pivotFirst = 29; + Kurtosis kurtosis = 30; } // Optional filter expression for SQL FILTER (WHERE ...) clause. @@ -266,6 +267,20 @@ message Stddev { StatisticsType stats_type = 4; } +// Excess kurtosis (Fisher definition: normal distribution -> 0). Spark's +// intermediate buffer is `[n, avg, m2, m3, m4]` of Float64 to match the +// `CentralMomentAgg` DeclarativeAggregate wire format for mixed +// Partial/Final execution with Spark. +message Kurtosis { + Expr child = 1; + bool null_on_divide_by_zero = 2; + // The mode of the `Divide` in Spark's final `n * m4 / (m2 * m2) - 3.0`, captured when the + // aggregate was built. A divisor that underflows to zero returns null in LEGACY mode and + // raises DIVIDE_BY_ZERO in ANSI mode. That is a different switch from + // `null_on_divide_by_zero`, which comes from `legacyStatisticalAggregate`. + EvalMode eval_mode = 3; +} + message Correlation { Expr child1 = 1; Expr child2 = 2; diff --git a/native/spark-expr/src/agg_funcs/kurtosis.rs b/native/spark-expr/src/agg_funcs/kurtosis.rs new file mode 100644 index 00000000000..014cb1d93d7 --- /dev/null +++ b/native/spark-expr/src/agg_funcs/kurtosis.rs @@ -0,0 +1,413 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Spark-compatible excess-kurtosis aggregate. +//! +//! Spark's `Kurtosis` is a `CentralMomentAgg` (`DeclarativeAggregate`) whose +//! intermediate buffer is `[n, avg, m2, m3, m4]` of Float64. This accumulator +//! mirrors that buffer exactly, using the same higher-order online update / +//! merge recurrences (Meng 2015) that `CentralMomentAgg` compiles into +//! catalyst expressions. Matching the wire format lets Spark's Partial and +//! Comet's Final (or vice versa) share intermediate state without a cast. +//! +//! Result formula (excess kurtosis, Fisher definition): +//! +//! * `n == 0` -> NULL +//! * `m2 == 0` -> NULL when `null_on_divide_by_zero`, else NaN +//! * otherwise -> `n * m4 / (m2 * m2) - 3.0` + +use std::mem::size_of; +use std::sync::Arc; + +use arrow::array::{ArrayRef, Float64Array}; +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion::common::{downcast_value, Result, ScalarValue}; +use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; +use datafusion::logical_expr::Volatility::Immutable; +use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature}; +use datafusion::physical_expr::expressions::format_state_name; + +use crate::agg_funcs::welford::{moments4_merge, moments4_update, ZeroDivisor}; +use crate::{EvalMode, QueryContextMap}; + +#[derive(Debug, PartialEq, Eq)] +pub struct Kurtosis { + name: String, + signature: Signature, + null_on_divide_by_zero: bool, + zero_divisor: ZeroDivisor, +} + +impl std::hash::Hash for Kurtosis { + fn hash(&self, state: &mut H) { + self.name.hash(state); + self.signature.hash(state); + self.null_on_divide_by_zero.hash(state); + self.zero_divisor.hash(state); + } +} + +impl Kurtosis { + pub fn new( + name: impl Into, + null_on_divide_by_zero: bool, + eval_mode: EvalMode, + expr_id: Option, + registry: Arc, + ) -> Self { + Self { + name: name.into(), + signature: Signature::numeric(1, Immutable), + null_on_divide_by_zero, + zero_divisor: ZeroDivisor::new(eval_mode, expr_id, registry), + } + } +} + +impl AggregateUDFImpl for Kurtosis { + fn name(&self) -> &str { + &self.name + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result> { + Ok(Box::new(KurtosisAccumulator::new( + self.null_on_divide_by_zero, + self.zero_divisor.clone(), + ))) + } + + // No `GroupsAccumulator`: grouped `kurtosis` deliberately runs through DataFusion's generic + // `GroupsAccumulatorAdapter`, which costs one boxed `Accumulator` and a `ScalarValue` round + // trip per group per batch. This is a gap relative to the neighbouring central-moment + // aggregates - `VarianceGroupsAccumulator` keeps flat `Vec` state and + // `StddevGroupsAccumulator` reuses it - and it is recorded here as a decision rather than an + // oversight. The vectorized version wants to land with `skewness`, since both are an + // `evaluate` over the same `[n, avg, m2, m3, m4]` state that `moments4_update` already + // maintains, and one flat-state accumulator should then serve all three. + + // Fields ordered to match Spark's `[n, avg, m2, m3, m4]` buffer so that a + // Spark-produced Partial state can be merged into a Comet-produced Final + // (and vice versa) without a schema conversion. + fn state_fields(&self, _args: StateFieldsArgs) -> Result> { + Ok(vec![ + Arc::new(Field::new( + format_state_name(&self.name, "n"), + DataType::Float64, + true, + )), + Arc::new(Field::new( + format_state_name(&self.name, "avg"), + DataType::Float64, + true, + )), + Arc::new(Field::new( + format_state_name(&self.name, "m2"), + DataType::Float64, + true, + )), + Arc::new(Field::new( + format_state_name(&self.name, "m3"), + DataType::Float64, + true, + )), + Arc::new(Field::new( + format_state_name(&self.name, "m4"), + DataType::Float64, + true, + )), + ]) + } +} + +#[derive(Debug)] +pub struct KurtosisAccumulator { + n: f64, + avg: f64, + m2: f64, + m3: f64, + m4: f64, + null_on_divide_by_zero: bool, + zero_divisor: ZeroDivisor, +} + +impl KurtosisAccumulator { + pub(crate) fn new(null_on_divide_by_zero: bool, zero_divisor: ZeroDivisor) -> Self { + Self { + n: 0.0, + avg: 0.0, + m2: 0.0, + m3: 0.0, + m4: 0.0, + null_on_divide_by_zero, + zero_divisor, + } + } +} + +impl Accumulator for KurtosisAccumulator { + fn state(&mut self) -> Result> { + Ok(vec![ + ScalarValue::from(self.n), + ScalarValue::from(self.avg), + ScalarValue::from(self.m2), + ScalarValue::from(self.m3), + ScalarValue::from(self.m4), + ]) + } + + fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { + let arr = downcast_value!(&values[0], Float64Array).iter().flatten(); + for value in arr { + let (n, avg, m2, m3, m4) = + moments4_update(self.n, self.avg, self.m2, self.m3, self.m4, value); + self.n = n; + self.avg = avg; + self.m2 = m2; + self.m3 = m3; + self.m4 = m4; + } + Ok(()) + } + + fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> { + let ns = downcast_value!(states[0], Float64Array); + let avgs = downcast_value!(states[1], Float64Array); + let m2s = downcast_value!(states[2], Float64Array); + let m3s = downcast_value!(states[3], Float64Array); + let m4s = downcast_value!(states[4], Float64Array); + + for i in 0..ns.len() { + // Even empty partials affect Spark's NaN propagation during merge. + let (n, avg, m2, m3, m4) = moments4_merge( + self.n, + self.avg, + self.m2, + self.m3, + self.m4, + ns.value(i), + avgs.value(i), + m2s.value(i), + m3s.value(i), + m4s.value(i), + ); + self.n = n; + self.avg = avg; + self.m2 = m2; + self.m3 = m3; + self.m4 = m4; + } + Ok(()) + } + + fn evaluate(&mut self) -> Result { + if self.n == 0.0 { + return Ok(ScalarValue::Float64(None)); + } + if self.m2 == 0.0 { + return Ok(ScalarValue::Float64(if self.null_on_divide_by_zero { + None + } else { + Some(f64::NAN) + })); + } + // Spark's guard above is on `m2`, but the division is by `m2 * m2`, and that product can + // underflow to zero while `m2` itself is finite and non-zero (`1e-100` and `2e-100` give + // an `m2` of 5e-201, whose square is 0). Spark's `Divide` then sees a zero divisor and + // applies its own rule, which is the eval mode it captured rather than + // `null_on_divide_by_zero`. Plain IEEE division here would return NaN instead. + let divisor = self.m2 * self.m2; + if divisor == 0.0 { + return match self.zero_divisor.error() { + Some(error) => Err(error), + None => Ok(ScalarValue::Float64(None)), + }; + } + Ok(ScalarValue::Float64(Some(self.n * self.m4 / divisor - 3.0))) + } + + fn size(&self) -> usize { + size_of::() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{create_query_context_map, QueryContext, SparkErrorWithContext}; + use datafusion::common::DataFusionError; + use datafusion_comet_common::SparkError; + + fn zero_divisor(eval_mode: EvalMode) -> ZeroDivisor { + ZeroDivisor::new(eval_mode, None, create_query_context_map()) + } + + fn accumulator( + values: &[f64], + null_on_divide_by_zero: bool, + zero_divisor: ZeroDivisor, + ) -> KurtosisAccumulator { + let mut acc = KurtosisAccumulator::new(null_on_divide_by_zero, zero_divisor); + let arr: ArrayRef = Arc::new(Float64Array::from(values.to_vec())); + acc.update_batch(&[arr]).unwrap(); + acc + } + + fn eval(values: &[f64], null_on_divide_by_zero: bool) -> Option { + let mut acc = accumulator( + values, + null_on_divide_by_zero, + zero_divisor(EvalMode::Legacy), + ); + match acc.evaluate().unwrap() { + ScalarValue::Float64(v) => v, + other => panic!("expected Float64, got {other:?}"), + } + } + + /// The accumulator's state as five single-row Float64 arrays, the shape `merge_batch` takes. + fn state_arrays(acc: &mut KurtosisAccumulator) -> Vec { + acc.state() + .unwrap() + .into_iter() + .map(|sv| match sv { + ScalarValue::Float64(v) => { + Arc::new(Float64Array::from(vec![v.unwrap()])) as ArrayRef + } + other => panic!("unexpected state scalar {other:?}"), + }) + .collect() + } + + #[test] + fn empty_group_returns_null() { + assert_eq!(eval(&[], true), None); + } + + #[test] + fn single_value_returns_divide_by_zero_result() { + // m2 == 0 with a single value => NULL when null_on_divide_by_zero, else NaN. + assert_eq!(eval(&[42.0], true), None); + let nan = eval(&[42.0], false).unwrap(); + assert!(nan.is_nan(), "expected NaN, got {nan}"); + } + + #[test] + fn matches_spark_example() { + // Spark's own example from ExpressionDescription: + // SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); + // => -0.7014368047529627 + let got = eval(&[-10.0, -20.0, 100.0, 1000.0], true).unwrap(); + assert!((got - -0.7014368047529627_f64).abs() < 1e-12, "got {got}"); + } + + #[test] + fn matches_spark_second_example() { + // SELECT kurtosis(col) FROM VALUES (1), (10), (100), (10), (1) as tab(col); + // => 0.19432323191699075 + let got = eval(&[1.0, 10.0, 100.0, 10.0, 1.0], true).unwrap(); + assert!((got - 0.19432323191699075_f64).abs() < 1e-12, "got {got}"); + } + + #[test] + fn merge_produces_same_result_as_single_batch() { + // Merging two partitions must reproduce the single-batch result. + let values = [-10.0_f64, -20.0, 100.0, 1000.0]; + let full = eval(&values, true).unwrap(); + + let mut a = accumulator(&values[..2], true, zero_divisor(EvalMode::Legacy)); + let mut b = accumulator(&values[2..], true, zero_divisor(EvalMode::Legacy)); + b.merge_batch(&state_arrays(&mut a)).unwrap(); + + let merged = match b.evaluate().unwrap() { + ScalarValue::Float64(Some(v)) => v, + other => panic!("expected Float64(Some(_)), got {other:?}"), + }; + assert!((merged - full).abs() < 1e-9, "merged={merged}, full={full}"); + } + + /// Spark merges an empty partial like any other, and with values this large the merge + /// overflows to NaN. Skipping the empty partial returned a finite -1.22 instead. + #[test] + fn merging_an_empty_partial_keeps_spark_nan_propagation() { + let values: Vec = (0..10).map(|i| 5e77 * (1.0 + i as f64 * 1e-9)).collect(); + let mut data = accumulator(&values, true, zero_divisor(EvalMode::Legacy)); + let mut empty = accumulator(&[], true, zero_divisor(EvalMode::Legacy)); + + let mut merged = accumulator(&[], true, zero_divisor(EvalMode::Legacy)); + merged.merge_batch(&state_arrays(&mut data)).unwrap(); + merged.merge_batch(&state_arrays(&mut empty)).unwrap(); + + let ScalarValue::Float64(Some(got)) = merged.evaluate().unwrap() else { + panic!("expected a Float64 value"); + }; + assert!(got.is_nan(), "expected NaN, got {got}"); + } + + /// `m2` is finite and non-zero here, so Spark's `m2 === 0` guard does not fire, but + /// `m2 * m2` underflows to zero and Spark's `Divide` takes over. That means the eval mode + /// decides, not `null_on_divide_by_zero`. Plain IEEE division returned NaN. + #[test] + fn divisor_underflow_follows_spark_division_semantics() { + let values = [1e-100, 2e-100]; + + // Confirm the premise rather than assuming it: m2 != 0 but m2 * m2 == 0. + let probe = accumulator(&values, true, zero_divisor(EvalMode::Legacy)); + assert_ne!(probe.m2, 0.0, "m2 must be non-zero for this case to bite"); + assert_eq!(probe.m2 * probe.m2, 0.0, "m2 * m2 must underflow to zero"); + + // LEGACY: NULL, for either value of null_on_divide_by_zero, because this path is + // governed by the Divide and not by the m2 == 0 branch. + for null_on_divide_by_zero in [true, false] { + assert_eq!(eval(&values, null_on_divide_by_zero), None); + } + + // ANSI: DIVIDE_BY_ZERO. + let mut ansi = accumulator(&values, true, zero_divisor(EvalMode::Ansi)); + let err = ansi.evaluate().unwrap_err().to_string(); + assert!( + err.contains("DIVIDE_BY_ZERO"), + "expected a DIVIDE_BY_ZERO error, got {err}" + ); + } + + #[test] + fn ansi_error_carries_query_context() { + let sql = "SELECT kurtosis(v) FROM t"; + let registry = create_query_context_map(); + registry.register( + 7, + QueryContext::new(sql.to_string(), 7, 17, None, None, 1, 7), + ); + let zero_divisor = ZeroDivisor::new(EvalMode::Ansi, Some(7), registry); + let mut acc = accumulator(&[1e-100, 2e-100], true, zero_divisor); + + let Err(DataFusionError::External(error)) = acc.evaluate() else { + panic!("expected a Spark error with query context"); + }; + let error = error.downcast_ref::().unwrap(); + assert!(matches!(error.error, SparkError::DivideByZero)); + assert_eq!(error.context.as_ref().unwrap().sql_text.as_str(), sql); + } +} diff --git a/native/spark-expr/src/agg_funcs/mod.rs b/native/spark-expr/src/agg_funcs/mod.rs index 7133a9cff90..4bd6c54bb2f 100644 --- a/native/spark-expr/src/agg_funcs/mod.rs +++ b/native/spark-expr/src/agg_funcs/mod.rs @@ -26,6 +26,7 @@ mod hll_plus_plus_const; mod hll_sketch; mod hll_sketch_agg; mod hll_union_agg; +mod kurtosis; mod list_agg; mod max_min_by; mod min_max; @@ -51,6 +52,7 @@ pub use hll_plus_plus::{hllpp_precision, HllPlusPlus}; pub use hll_sketch::{estimate_from_bytes, SparkHllSketch, SparkHllUnion}; pub use hll_sketch_agg::HllSketchAgg; pub use hll_union_agg::HllUnionAgg; +pub use kurtosis::Kurtosis; pub use list_agg::SparkListAgg; pub use max_min_by::MaxMinBy; pub use min_max::SparkMinMax; diff --git a/native/spark-expr/src/agg_funcs/welford.rs b/native/spark-expr/src/agg_funcs/welford.rs index 44c096f5223..8eaf9fb01c1 100644 --- a/native/spark-expr/src/agg_funcs/welford.rs +++ b/native/spark-expr/src/agg_funcs/welford.rs @@ -28,9 +28,10 @@ use std::sync::Arc; use crate::{divide_by_zero_error, EvalMode, QueryContextMap, SparkErrorWithContext}; -/// How the `Divide` in Spark's `ck / sqrt(xMk * yMk)`, shared by `corr` and -/// `regr_r2`, treats a zero denominator. It returns null in LEGACY mode and -/// raises DIVIDE_BY_ZERO, carrying the aggregate call's query context, in ANSI mode. +/// How a `Divide` in a statistical aggregate treats a zero denominator: `ck / sqrt(xMk * yMk)`, +/// shared by `corr` and `regr_r2`, and `n * m4 / (m2 * m2) - 3.0` in `kurtosis`. It returns null +/// in LEGACY mode and raises DIVIDE_BY_ZERO, carrying the aggregate call's query context, in ANSI +/// mode. #[derive(Debug, Clone)] pub(crate) struct ZeroDivisor { eval_mode: EvalMode, @@ -251,3 +252,63 @@ pub(crate) fn covariance_merge( let new_c = c_a + c_b + delta1 * delta2_n * count_a * count_b; (new_count, new_mean1, new_mean2, new_c) } + +/// Online update for the first four central moments `[n, avg, m2, m3, m4]`. Direct port of +/// Spark's `CentralMomentAgg.updateExpressionsDef` for `momentOrder = 4`. +/// +/// The order-4 recurrence subsumes the order-2 one above, so `skewness` (`momentOrder = 3`) +/// is an `evaluate` on top of this same state rather than another copy of the algebra. +#[inline] +pub(crate) fn moments4_update( + n: f64, + avg: f64, + m2: f64, + m3: f64, + m4: f64, + value: f64, +) -> (f64, f64, f64, f64, f64) { + let new_n = n + 1.0; + let delta = value - avg; + let delta_n = delta / new_n; + let new_avg = avg + delta_n; + let new_m2 = m2 + delta * (delta - delta_n); + let delta2 = delta * delta; + let delta_n2 = delta_n * delta_n; + let new_m3 = m3 - 3.0 * delta_n * new_m2 + delta * (delta2 - delta_n2); + let new_m4 = m4 - 4.0 * delta_n * new_m3 - 6.0 * delta_n2 * new_m2 + + delta * (delta * delta2 - delta_n * delta_n2); + (new_n, new_avg, new_m2, new_m3, new_m4) +} + +/// Merge two partial `[n, avg, m2, m3, m4]` states. Direct port of Spark's +/// `CentralMomentAgg.mergeExpressions` for `momentOrder = 4`. +#[inline] +#[allow(clippy::too_many_arguments)] +pub(crate) fn moments4_merge( + n1: f64, + avg1: f64, + m2_1: f64, + m3_1: f64, + m4_1: f64, + n2: f64, + avg2: f64, + m2_2: f64, + m3_2: f64, + m4_2: f64, +) -> (f64, f64, f64, f64, f64) { + let new_n = n1 + n2; + let delta = avg2 - avg1; + let delta_n = if new_n == 0.0 { 0.0 } else { delta / new_n }; + let new_avg = avg1 + delta_n * n2; + let new_m2 = m2_1 + m2_2 + delta * delta_n * n1 * n2; + let new_m3 = m3_1 + + m3_2 + + delta_n * delta_n * delta * n1 * n2 * (n1 - n2) + + 3.0 * delta_n * (n1 * m2_2 - n2 * m2_1); + let new_m4 = m4_1 + + m4_2 + + delta_n * delta_n * delta_n * delta * n1 * n2 * (n1 * n1 - n1 * n2 + n2 * n2) + + 6.0 * delta_n * delta_n * (n1 * n1 * m2_2 + n2 * n2 * m2_1) + + 4.0 * delta_n * (n1 * m3_2 - n2 * m3_1); + (new_n, new_avg, new_m2, new_m3, new_m4) +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index c8b89b0742c..c04dbfba779 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -432,6 +432,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[CovPopulation] -> CometCovPopulation, classOf[CovSample] -> CometCovSample, classOf[First] -> CometFirst, + classOf[Kurtosis] -> CometKurtosis, classOf[Last] -> CometLast, classOf[Max] -> CometMax, classOf[MaxBy] -> CometMaxBy, diff --git a/spark/src/main/scala/org/apache/comet/serde/aggregates.scala b/spark/src/main/scala/org/apache/comet/serde/aggregates.scala index 15318da2413..ea85933ee33 100644 --- a/spark/src/main/scala/org/apache/comet/serde/aggregates.scala +++ b/spark/src/main/scala/org/apache/comet/serde/aggregates.scala @@ -23,7 +23,7 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Divide, Expression, Literal} -import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, ApproximatePercentile, Average, BitAndAgg, BitOrAgg, BitXorAgg, BloomFilterAggregate, CentralMomentAgg, CollectList, CollectSet, Complete, Corr, Count, Covariance, CovPopulation, CovSample, DeclarativeAggregate, First, HyperLogLogPlusPlus, Last, Max, MaxBy, MaxMinBy, Min, MinBy, Mode, Partial, PartialMerge, Percentile, PivotFirst, RegrIntercept, RegrR2, RegrReplacement, RegrSlope, RegrSXY, StddevPop, StddevSamp, Sum, VariancePop, VarianceSamp} +import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, ApproximatePercentile, Average, BitAndAgg, BitOrAgg, BitXorAgg, BloomFilterAggregate, CentralMomentAgg, CollectList, CollectSet, Complete, Corr, Count, Covariance, CovPopulation, CovSample, DeclarativeAggregate, First, HyperLogLogPlusPlus, Kurtosis, Last, Max, MaxBy, MaxMinBy, Min, MinBy, Mode, Partial, PartialMerge, Percentile, PivotFirst, RegrIntercept, RegrR2, RegrReplacement, RegrSlope, RegrSXY, StddevPop, StddevSamp, Sum, VariancePop, VarianceSamp} import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.comet.CometExecUtils import org.apache.spark.sql.internal.SQLConf @@ -1377,6 +1377,58 @@ object CometApproxCountDistinct extends CometAggregateExpressionSerde[HyperLogLo } } +object CometKurtosis extends CometAggregateExpressionSerde[Kurtosis] { + + // Not marked safe for mixed partial/final: follows the same policy as `Variance` / `Stddev`, + // whose complex `[n, avg, m2, ...]` buffer is not certified compatible across engines. The + // native accumulator does mirror Spark's `[n, avg, m2, m3, m4]` wire format, so lifting this + // to `true` should be considered together with the other `CentralMomentAgg` serdes. + + override def convert( + aggExpr: AggregateExpression, + kurtosis: Kurtosis, + inputs: Seq[Attribute], + binding: Boolean, + conf: SQLConf): Option[ExprOuterClass.AggExpr] = { + // Spark's evaluate expression divides by `m2 * m2`. `m2` can be non-zero while `m2 * m2` + // underflows to zero, which the `m2 === 0` guard above it does not catch, so the native side + // needs to know whether that divisor should raise or return null. That `Divide` captured its + // mode when the aggregate was built, which can differ from the session's now (a view + // analyzed under another ANSI setting), so read the captured mode. + CapturedEvalMode.ofDivide(kurtosis) match { + case Some(evalMode) => convertKurtosis(aggExpr, kurtosis, inputs, binding, evalMode) + case None => + withFallbackReason(aggExpr, "KURTOSIS division evaluation mode not supported") + None + } + } + + private def convertKurtosis( + aggExpr: AggregateExpression, + kurtosis: Kurtosis, + inputs: Seq[Attribute], + binding: Boolean, + evalMode: CometEvalMode.Value): Option[ExprOuterClass.AggExpr] = { + val childExpr = exprToProto(kurtosis.child, inputs, binding) + + if (childExpr.isDefined) { + val builder = ExprOuterClass.Kurtosis.newBuilder() + builder.setChild(childExpr.get) + builder.setNullOnDivideByZero(kurtosis.nullOnDivideByZero) + builder.setEvalMode(evalModeToProto(evalMode)) + + Some( + ExprOuterClass.AggExpr + .newBuilder() + .setKurtosis(builder) + .build()) + } else { + withFallbackReason(aggExpr, "Child expression or data type not supported") + None + } + } +} + object CometMode extends CometAggregateExpressionSerde[Mode] with CometTypeShim { private val tieBreakReason = diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis.sql new file mode 100644 index 00000000000..159895aac48 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis.sql @@ -0,0 +1,289 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- ============================================================ +-- Setup +-- ============================================================ + +statement +CREATE TABLE k_dbl(v double, grp string) USING parquet + +statement +INSERT INTO k_dbl VALUES + (-10.0, 'g1'), (-20.0, 'g1'), (100.0, 'g1'), (1000.0, 'g1'), + (1.0, 'g2'), (10.0, 'g2'), (100.0, 'g2'), (10.0, 'g2'), (1.0, 'g2'), + (42.0, 'g3'), + (NULL, 'g4'), (NULL, 'g4'), + (7.0, 'g5'), (7.0, 'g5'), (7.0, 'g5') + +statement +CREATE TABLE k_int(v int, grp string) USING parquet + +statement +INSERT INTO k_int VALUES + (1, 'g1'), (10, 'g1'), (100, 'g1'), (10, 'g1'), (1, 'g1'), + (NULL, 'g2'), (5, 'g2') + +statement +CREATE TABLE k_dec(v decimal(10,2), grp string) USING parquet + +statement +INSERT INTO k_dec VALUES + (1.50, 'g1'), (2.50, 'g1'), (3.50, 'g1'), (4.50, 'g1') + +statement +CREATE TABLE k_empty(v double) USING parquet + +statement +CREATE TABLE k_spark_ex1(v double) USING parquet + +statement +INSERT INTO k_spark_ex1 VALUES (-10.0), (-20.0), (100.0), (1000.0) + +statement +CREATE TABLE k_spark_ex2(v double) USING parquet + +statement +INSERT INTO k_spark_ex2 VALUES (1.0), (10.0), (100.0), (10.0), (1.0) + +statement +CREATE TABLE k_single(v double) USING parquet + +statement +INSERT INTO k_single VALUES (42.0) + +statement +CREATE TABLE k_const(v double) USING parquet + +statement +INSERT INTO k_const VALUES (7.0), (7.0), (7.0) + +statement +CREATE TABLE k_lit(x int) USING parquet + +statement +INSERT INTO k_lit VALUES (1) + +-- ============================================================ +-- Spark's own example: matches -0.7014368047529627. +-- ============================================================ + +query +SELECT kurtosis(v) FROM k_spark_ex1 + +-- Spark's second example: matches 0.19432323191699075. +query +SELECT kurtosis(v) FROM k_spark_ex2 + +-- ============================================================ +-- GROUP BY over doubles: covers a "normal" group (g1), a heavier +-- group (g2), a single-value group (g3, m2=0 => NULL by default), +-- an all-NULL group (g4 => NULL), and constants (g5, m2=0). +-- ============================================================ + +query +SELECT grp, kurtosis(v) FROM k_dbl GROUP BY grp ORDER BY grp + +-- ============================================================ +-- Global aggregate (no GROUP BY). +-- ============================================================ + +query +SELECT kurtosis(v) FROM k_dbl + +-- Empty table returns NULL. +query +SELECT kurtosis(v) FROM k_empty + +-- ============================================================ +-- Integer input: promoted to Double by Spark's ImplicitCastInputTypes. +-- ============================================================ + +query +SELECT grp, kurtosis(v) FROM k_int GROUP BY grp ORDER BY grp + +-- ============================================================ +-- Decimal input. +-- ============================================================ + +query +SELECT grp, kurtosis(v) FROM k_dec GROUP BY grp ORDER BY grp + +-- ============================================================ +-- Literal argument (constant folded; still exercises planning). +-- ============================================================ + +query +SELECT kurtosis(1.0) FROM k_lit + +query +SELECT kurtosis(NULL) FROM k_lit + +-- ============================================================ +-- Divide-by-zero cases under default (nullOnDivideByZero=true): +-- single-value and all-equal groups both yield NULL. See +-- kurtosis_legacy.sql for the `legacyStatisticalAggregate=true` +-- variant that returns NaN instead. +-- ============================================================ + +query +SELECT kurtosis(v) FROM k_single + +query +SELECT kurtosis(v) FROM k_const + +-- ============================================================ +-- FILTER (WHERE ...) — Partial only carries the filter. +-- ============================================================ + +query +SELECT grp, kurtosis(v) FILTER (WHERE v > 0) FROM k_dbl GROUP BY grp ORDER BY grp + +-- ============================================================ +-- Additional coverage requested by the audit. +-- ============================================================ + +statement +CREATE TABLE k_float(v float, grp string) USING parquet + +statement +INSERT INTO k_float VALUES + (CAST(1.0 AS FLOAT), 'g1'), (CAST(10.0 AS FLOAT), 'g1'), + (CAST(100.0 AS FLOAT), 'g1'), (CAST(10.0 AS FLOAT), 'g1'), + (CAST(1.0 AS FLOAT), 'g1') + +statement +CREATE TABLE k_long(v bigint, grp string) USING parquet + +statement +INSERT INTO k_long VALUES + (1, 'g1'), (10, 'g1'), (100, 'g1'), (10, 'g1'), (1, 'g1') + +statement +CREATE TABLE k_wnd(k int, part string, v double) USING parquet + +statement +INSERT INTO k_wnd VALUES + (1, 'p1', 1.0), (2, 'p1', 1.0), (3, 'p1', 2.0), (4, 'p1', 2.0), + (5, 'p1', 3.0), (6, 'p1', 3.0), (7, 'p1', 3.0), + (8, 'p2', 1.0), (9, 'p2', 2.0), (10, 'p2', 5.0) + +-- Float input: promoted to Double by Spark's ImplicitCastInputTypes. +query +SELECT grp, kurtosis(v) FROM k_float GROUP BY grp ORDER BY grp + +-- BigInt input. +query +SELECT grp, kurtosis(v) FROM k_long GROUP BY grp ORDER BY grp + +-- Mixed with other CentralMomentAgg siblings in one query. +query +SELECT kurtosis(v), avg(v), stddev(v), count(*) FROM k_dbl WHERE v IS NOT NULL + +-- Window use: matches Spark's `DataFrameWindowFunctionsSuite` +-- "skewness and kurtosis functions in window" test. Comet's window +-- path doesn't wire `kurtosis` as a window aggregate today, so this +-- falls back to Spark. +query expect_fallback(is not supported for window function) +SELECT k, + kurtosis(v) OVER (PARTITION BY part ORDER BY k + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) +FROM k_wnd ORDER BY k + +-- ============================================================ +-- Numerical stress: NaN/Infinity/-Infinity in a group. Spark +-- propagates these through arithmetic, so the aggregate is +-- expected to produce NaN or NULL rather than a finite value. +-- ============================================================ + +statement +CREATE TABLE k_nan(v double) USING parquet + +statement +INSERT INTO k_nan VALUES + (1.0), (2.0), (CAST('NaN' AS DOUBLE)), (3.0) + +statement +CREATE TABLE k_inf(v double) USING parquet + +statement +INSERT INTO k_inf VALUES + (1.0), (2.0), (CAST('Infinity' AS DOUBLE)), (3.0) + +statement +CREATE TABLE k_neg_inf(v double) USING parquet + +statement +INSERT INTO k_neg_inf VALUES + (1.0), (2.0), (CAST('-Infinity' AS DOUBLE)), (3.0) + +query +SELECT kurtosis(v) FROM k_nan + +query +SELECT kurtosis(v) FROM k_inf + +query +SELECT kurtosis(v) FROM k_neg_inf + +-- ============================================================ +-- Large-magnitude inputs. +-- ============================================================ + +statement +CREATE TABLE k_big(v double) USING parquet + +statement +INSERT INTO k_big VALUES + (1.0e15), (2.0e15), (3.0e15), (4.0e15), (5.0e15) + +query +SELECT kurtosis(v) FROM k_big + +-- The division by m2 * m2 follows the ANSI setting the aggregate was analyzed under, not the one +-- it runs under. 1e-100 and 2e-100 give an m2 whose square underflows to zero. A view keeps the +-- aggregate it analyzed. +statement +CREATE TABLE test_kurtosis_underflow(v double) USING parquet + +statement +INSERT INTO test_kurtosis_underflow VALUES (1e-100D), (2e-100D) + +statement +SET spark.sql.ansi.enabled=false + +statement +CREATE OR REPLACE TEMPORARY VIEW kurtosis_analyzed_legacy AS SELECT kurtosis(v) AS k FROM test_kurtosis_underflow + +statement +SET spark.sql.ansi.enabled=true + +statement +CREATE OR REPLACE TEMPORARY VIEW kurtosis_analyzed_ansi AS SELECT kurtosis(v) AS k FROM test_kurtosis_underflow + +-- Analyzed with ANSI off and run with it on: NULL. +query +SELECT k FROM kurtosis_analyzed_legacy + +statement +SET spark.sql.ansi.enabled=false + +-- Analyzed with ANSI on and run with it off: raises. +query expect_error_class(DIVIDE_BY_ZERO) +SELECT k FROM kurtosis_analyzed_ansi diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis_legacy.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis_legacy.sql new file mode 100644 index 00000000000..28e3424ac49 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis_legacy.sql @@ -0,0 +1,41 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Legacy statistical-aggregate semantics: divide-by-zero returns NaN +-- instead of NULL. Spark passes `!legacyStatisticalAggregate` as +-- `nullOnDivideByZero` when constructing the aggregate. +-- Config: spark.sql.legacy.statisticalAggregate=true + +statement +CREATE TABLE k_legacy_single(v double) USING parquet + +statement +INSERT INTO k_legacy_single VALUES (42.0) + +statement +CREATE TABLE k_legacy_const(v double) USING parquet + +statement +INSERT INTO k_legacy_const VALUES (7.0), (7.0), (7.0) + +-- Single-value group: m2 == 0. Expect NaN (not NULL). +query +SELECT kurtosis(v) FROM k_legacy_single + +-- All-equal group: same divide-by-zero shape. +query +SELECT kurtosis(v) FROM k_legacy_const diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis_partials.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis_partials.sql new file mode 100644 index 00000000000..eb9817479ea --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/kurtosis_partials.sql @@ -0,0 +1,65 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Each INSERT coalesces to one task, so it writes one file, and an open cost as large as a split +-- keeps every file in its own partition. Every query below then merges two partials, and plain +-- `query` compares the merged result with Spark's exactly. +-- Config: spark.sql.files.openCostInBytes=134217728 + +-- Ordinary values in both files. Group 3 is NULL in the first file, so its first partial is +-- empty. Merging these partials rounds differently if `moments4_merge` reorders the arithmetic +-- for `avg`, `m2` or `m4`, and every query gives the same result in either arrival order. + +statement +CREATE TABLE test_kurtosis_partials(g int, v double) USING parquet + +statement +INSERT INTO test_kurtosis_partials SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, -13.44), (1, -13.9), (1, 0.94), (1, -2.92), + (2, -19.6), (2, -2.6), (2, 11.5), + (3, NULL) AS t(g, v) + +statement +INSERT INTO test_kurtosis_partials SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 13.3), (1, 19.0), (1, 2.39), + (2, -13.3), (2, -15.54), (2, -3.4), + (3, 4.0), (3, 8.0), (3, 16.0), (3, 23.0) AS t(g, v) + +query +SELECT kurtosis(v) FROM test_kurtosis_partials + +query +SELECT g, kurtosis(v) FROM test_kurtosis_partials GROUP BY g + +-- The moments of these values overflow when merged, so Spark returns NaN. Merging the all-NULL +-- file's empty partial must not hide that. + +statement +CREATE TABLE test_kurtosis_empty_partial(g int, v double) USING parquet + +statement +INSERT INTO test_kurtosis_empty_partial SELECT /*+ COALESCE(1) */ 0, 5e77 * (1 + id * 1e-9) + FROM range(10) + +statement +INSERT INTO test_kurtosis_empty_partial SELECT /*+ COALESCE(1) */ 0, CAST(NULL AS double) + +query +SELECT kurtosis(v) FROM test_kurtosis_empty_partial + +query +SELECT g, kurtosis(v) FROM test_kurtosis_empty_partial GROUP BY g