Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 73 additions & 3 deletions ballista/scheduler/src/state/aqe/adapter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,18 @@
// under the License.

use crate::planner::create_shuffle_writer_with_config;
use crate::state::aqe::adaptive_rules::AdaptiveRules;
use crate::state::aqe::execution_plan::{
AdaptiveDatafusionExec, ExchangeExec, RangeRepartitionRouting,
};
use crate::state::aqe::optimizer_rule::coalesce_partitions::tolerates_partition_count_change;
use crate::state::aqe::planner::AdaptiveStageInfo;
use crate::state::aqe::skew_join::sort_mapper_outputs;
use crate::state::execution_graph::StageOutput;
use ballista_core::JobId;
use ballista_core::execution_plans::sort_shuffle::ShuffleRouting;
use ballista_core::execution_plans::{
OrderedRangeRepartitionExec, RangeFilterExec, RangeShuffleReaderExec,
CoalescePlan, OrderedRangeRepartitionExec, RangeFilterExec, RangeShuffleReaderExec,
RangeShuffleWriterExec, ShuffleReaderExec, ShuffleWriter, ShuffleWriterExec,
UnorderedRangeRepartitionExec,
};
Expand Down Expand Up @@ -105,6 +107,9 @@ fn use_range_shuffle_writer(
#[derive(Debug, Clone, Default)]
pub(crate) struct BallistaAdapter {
inputs: HashMap<usize, StageOutput>,
/// The scheduler embedder's rules, which rebuild operators of their own
/// over coalesced shuffles.
rules: Option<Arc<dyn AdaptiveRules>>,
}

///
Expand Down Expand Up @@ -227,6 +232,10 @@ impl BallistaAdapter {
&mut self,
plan: Arc<dyn ExecutionPlan>,
) -> datafusion::error::Result<Transformed<Arc<dyn ExecutionPlan>>> {
if let Some(rebuilt) = self.rebuild_over_coalesced_shuffles(&plan)? {
return Ok(Transformed::new(rebuilt, true, TreeNodeRecursion::Jump));
}

// A merge with a row limit on top of an exchange: build the reader
// with the limit already set. Both merge on the same ordering, so the
// consumer's first `n` rows come from the reader's first `n`.
Expand All @@ -248,16 +257,57 @@ impl BallistaAdapter {
Ok(Transformed::no(plan))
}

/// Rebuilds `plan`, an operator of the embedder's own above a coalesced
/// shuffle of this stage, with its [`AdaptiveRules::with_coalesced_children`],
/// over its children adapted as usual. `None` for any other node.
fn rebuild_over_coalesced_shuffles(
&mut self,
plan: &Arc<dyn ExecutionPlan>,
) -> datafusion::error::Result<Option<Arc<dyn ExecutionPlan>>> {
let Some(rules) = self.rules.clone() else {
return Ok(None);
};
if plan.is::<ExchangeExec>()
|| tolerates_partition_count_change(plan.as_ref())
|| !rules.tolerates_partition_count_change(plan.as_ref())
{
return Ok(None);
}
let Some(coalesce) = coalesce_of_stage_inputs(plan) else {
return Ok(None);
};
let children = plan
.children()
.into_iter()
.map(|child| {
Arc::clone(child)
.transform_down(|node| self.transform_children(node))
.map(|transformed| transformed.data)
})
.collect::<datafusion::error::Result<Vec<_>>>()?;
match rules.with_coalesced_children(plan, children.clone(), &coalesce) {
Some(rebuilt) => rebuilt.map(Some),
None => replace_children_if_necessary(Arc::clone(plan), children).map(Some),
}
}

/// Converts Adaptive plan to plan which ballista expects
/// This is to be used to convert [ExchangeExec] to
/// ShuffleWriterExec/SortShuffleWriterExec and [ShuffleReaderExec]
///
/// `rules` are the scheduler embedder's, which rebuild operators of their
/// own over coalesced shuffles.
pub fn adapt_to_ballista(
plan: Arc<dyn ExecutionPlan>,
job_id: &JobId,
config: &ConfigOptions,
rules: Option<Arc<dyn AdaptiveRules>>,
) -> datafusion::error::Result<AdaptiveStageInfo> {
let mut adapter = BallistaAdapter {
rules,
..Default::default()
};
if let Some(root) = plan.downcast_ref::<ExchangeExec>() {
let mut adapter = BallistaAdapter::default();
resolve_range_filter_cuts(root.input())?;
let plan = root
.input()
Expand Down Expand Up @@ -287,7 +337,6 @@ impl BallistaAdapter {
inputs: adapter.inputs,
})
} else if let Some(root) = plan.downcast_ref::<AdaptiveDatafusionExec>() {
let mut adapter = BallistaAdapter::default();
resolve_range_filter_cuts(root.input())?;
let plan = root
.input()
Expand Down Expand Up @@ -317,6 +366,27 @@ impl BallistaAdapter {
}
}

/// The coalesce decision on the shuffles `plan` reads in its stage, which
/// [`CoalescePartitionsRule`](crate::state::aqe::optimizer_rule::CoalescePartitionsRule)
/// shares across all of them, or `None` if they are read as written.
fn coalesce_of_stage_inputs(plan: &Arc<dyn ExecutionPlan>) -> Option<Arc<CoalescePlan>> {
let mut coalesce = None;
let _ = plan.apply_children(|child| {
child.apply(|node| {
let Some(exchange) = node.downcast_ref::<ExchangeExec>() else {
return Ok(TreeNodeRecursion::Continue);
};
coalesce = exchange.coalesce();
Ok(if coalesce.is_some() {
TreeNodeRecursion::Stop
} else {
TreeNodeRecursion::Jump
})
})
});
coalesce
}

/// Hand every [`RangeShuffleReaderExec`] the value range its output
/// partitions cover, taken from the [`RangeFilterExec`] above it.
///
Expand Down
42 changes: 42 additions & 0 deletions ballista/scheduler/src/state/aqe/adaptive_rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@

//! Rules a scheduler embedder adds to adaptive query planning.

use ballista_core::execution_plans::CoalescePlan;
use datafusion::common::Result;
use datafusion::physical_optimizer::PhysicalOptimizerRule;
use datafusion::physical_plan::ExecutionPlan;
use std::fmt::Debug;
use std::sync::Arc;
use std::sync::atomic::AtomicUsize;
Expand Down Expand Up @@ -52,4 +55,43 @@ pub trait AdaptiveRules: Debug + Send + Sync {
fn replan_rules(&self, _plan_ids: Arc<AtomicUsize>) -> Vec<AdaptiveRule> {
vec![]
}

/// Whether `plan`, a node of your own between a stage's root and the
/// shuffles it reads, produces the same rows when those shuffles are read
/// with fewer, coalesced partitions (`ballista.planner.coalesce.enabled`).
///
/// Coalescing groups neighbouring upstream partitions with one mapping
/// for every shuffle the stage reads, so hash co-partitioning is kept. A
/// node that only needs that, such as a join or an aggregate over hash
/// partitioned inputs, tolerates it. A node that depends on the partition
/// count or index itself does not. The default, `false`, leaves every
/// stage with such a node uncoalesced.
///
/// Asked only about nodes Ballista doesn't know, as the plan is
/// optimized after each stage completes.
fn tolerates_partition_count_change(&self, _plan: &dyn ExecutionPlan) -> bool {
false
}

/// Rebuilds `plan`, a node [`Self::tolerates_partition_count_change`]
/// accepted, over `children` once the shuffles below it are read as
/// `coalesce` describes, or returns `None` to rebuild it with
/// [`ExecutionPlan::with_new_children`].
///
/// A node that keeps per-partition state of its own beside its inputs,
/// such as a child with one row per partition, rebuilds that state for
/// `coalesce.groups.len()` partitions here. Upstream partition `i` is
/// read by the output partition whose group lists `i`.
///
/// `children` are already rebuilt, with every shuffle reader in them
/// coalesced. Called when the stage is turned into the plan its tasks
/// run.
fn with_coalesced_children(
&self,
_plan: &Arc<dyn ExecutionPlan>,
_children: Vec<Arc<dyn ExecutionPlan>>,
_coalesce: &CoalescePlan,
) -> Option<Result<Arc<dyn ExecutionPlan>>> {
None
}
}
54 changes: 36 additions & 18 deletions ballista/scheduler/src/state/aqe/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ use ballista_core::execution_plans::{
PartitionedBoundedWindowAggExec, PrefixMergeExec, RangeFilterExec, cut_partitions,
merge_runtime_stats_reports, prefix_merge_window_state, repartition_routing_expr,
};
use ballista_core::serde::BallistaPhysicalExtensionCodec;
use ballista_core::serde::protobuf::failed_task::FailedReason;
use ballista_core::serde::protobuf::job_status::Status;
use ballista_core::serde::protobuf::{
Expand Down Expand Up @@ -151,28 +152,48 @@ pub(crate) struct AdaptiveExecutionGraph {
logical_plan: Option<String>,
}

/// What the scheduler, rather than the job's session, adds to planning a job
/// adaptively.
#[derive(Debug, Clone)]
pub(crate) struct SchedulerExtensions {
/// The codec the scheduler ships stages with, which exchange reuse
/// matches stages on.
pub(crate) physical_codec: Arc<dyn PhysicalExtensionCodec>,
/// See [`SchedulerConfig::adaptive_rules`](crate::config::SchedulerConfig::adaptive_rules).
pub(crate) adaptive_rules: Option<Arc<dyn AdaptiveRules>>,
}

impl Default for SchedulerExtensions {
fn default() -> Self {
Self {
physical_codec: Arc::new(BallistaPhysicalExtensionCodec::default()),
adaptive_rules: None,
}
}
}

impl AdaptiveExecutionGraph {
/// Creates a new `ExecutionGraph` from a physical execution plan.
///
/// This will use the `DistributedPlanner` to break the plan into stages
/// and build the DAG structure needed for distributed execution.
///
/// `physical_codec` is the codec the scheduler ships stages with, which
/// exchange reuse matches stages on.
#[allow(clippy::too_many_arguments)]
pub async fn try_new(
pub(crate) async fn try_new(
scheduler_id: &str,
job_id: &JobId,
job_name: &str,
ctx: &SessionContext,
logical_plan: &LogicalPlan,
queued_at: u64,
rules: Option<&dyn AdaptiveRules>,
physical_codec: Arc<dyn PhysicalExtensionCodec>,
extensions: SchedulerExtensions,
) -> ballista_core::error::Result<Self> {
let planner = AdaptivePlanner::try_new(ctx, logical_plan, job_id.clone(), rules)
.await?
.with_physical_codec(physical_codec);
let planner = AdaptivePlanner::try_new(
ctx,
logical_plan,
job_id.clone(),
extensions.adaptive_rules,
)
.await?
.with_physical_codec(extensions.physical_codec);
let logical_plan = Some(logical_plan.display_indent().to_string());

Self::try_new_with_planner(
Expand All @@ -189,26 +210,23 @@ impl AdaptiveExecutionGraph {
/// Creates a new `ExecutionGraph` from an already-built physical plan.
///
/// The plan is prepared and staged by [`AdaptivePlanner::try_new_from_physical`];
/// the graph has no logical plan to display. `physical_codec` is as in
/// [`Self::try_new`].
#[allow(clippy::too_many_arguments)]
pub fn try_new_from_physical(
/// the graph has no logical plan to display.
pub(crate) fn try_new_from_physical(
scheduler_id: &str,
job_id: &JobId,
job_name: &str,
ctx: &SessionContext,
physical_plan: Arc<dyn ExecutionPlan>,
queued_at: u64,
rules: Option<&dyn AdaptiveRules>,
physical_codec: Arc<dyn PhysicalExtensionCodec>,
extensions: SchedulerExtensions,
) -> ballista_core::error::Result<Self> {
let planner = AdaptivePlanner::try_new_from_physical(
ctx,
physical_plan,
job_id.clone(),
rules,
extensions.adaptive_rules,
)?
.with_physical_codec(physical_codec);
.with_physical_codec(extensions.physical_codec);

Self::try_new_with_planner(
scheduler_id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ use datafusion::physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};
use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties};
use log::debug;

use crate::state::aqe::adaptive_rules::AdaptiveRules;
use crate::state::aqe::coalesce::{
split_size_list_by_target_size, start_indices_to_partition_groups,
};
Expand All @@ -160,7 +161,19 @@ use crate::state::aqe::execution_plan::{
///
/// See module docs for design intent.
#[derive(Debug, Default)]
pub struct CoalescePartitionsRule;
pub struct CoalescePartitionsRule {
/// Asked about operators Ballista doesn't know, see
/// [`AdaptiveRules::tolerates_partition_count_change`].
rules: Option<Arc<dyn AdaptiveRules>>,
}

impl CoalescePartitionsRule {
/// A rule that asks `rules` whether operators Ballista doesn't know
/// tolerate a coalesced partition count.
pub fn new(rules: Option<Arc<dyn AdaptiveRules>>) -> Self {
Self { rules }
}
}

impl PhysicalOptimizerRule for CoalescePartitionsRule {
fn optimize(
Expand Down Expand Up @@ -213,7 +226,7 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule {
// checks every operator between the stage root and a leaf, since each
// of them runs over the coalesced partition count.
let mut leaves: Vec<Arc<dyn ExecutionPlan>> = Vec::new();
if !collect_alignment_group(&input, &mut leaves) {
if !collect_alignment_group(&input, self.rules.as_deref(), &mut leaves) {
debug!(
"[coalesce-rule] an operator of unknown partition-count sensitivity consumes a leaf; bail"
);
Expand Down Expand Up @@ -369,10 +382,12 @@ impl PhysicalOptimizerRule for CoalescePartitionsRule {
/// Collects the leaf `ExchangeExec`s under `node` into `leaves`.
///
/// Returns `false` when an operator between `node` and one of those leaves
/// is not known to [`tolerates_partition_count_change`]: the rule then has no
/// basis for changing the partition count that operator sees.
/// is neither on [`tolerates_partition_count_change`]'s list nor accepted by
/// the embedder's `rules`: the rule then has no basis for changing the
/// partition count that operator sees.
fn collect_alignment_group(
node: &Arc<dyn ExecutionPlan>,
rules: Option<&dyn AdaptiveRules>,
leaves: &mut Vec<Arc<dyn ExecutionPlan>>,
) -> bool {
if node.is::<ExchangeExec>() {
Expand All @@ -381,12 +396,15 @@ fn collect_alignment_group(
}
let leaves_before = leaves.len();
for child in node.children() {
if !collect_alignment_group(child, leaves) {
if !collect_alignment_group(child, rules, leaves) {
return false;
}
}
let consumes_leaf = leaves.len() > leaves_before;
!consumes_leaf || tolerates_partition_count_change(node.as_ref())
!consumes_leaf
|| tolerates_partition_count_change(node.as_ref())
|| rules
.is_some_and(|rules| rules.tolerates_partition_count_change(node.as_ref()))
}

/// Operators that produce the same rows over any partitioning of their
Expand All @@ -396,9 +414,11 @@ fn collect_alignment_group(
/// `ExecutionPlan` has no way to declare this, so the list is maintained by
/// hand, like the whitelists in `ballista_core::execution_plans::plan_algebra`.
/// Anything else, in particular an extension operator from a client that
/// submitted its own physical plan, is assumed to depend on the count.
/// submitted its own physical plan, is assumed to depend on the count unless
/// the scheduler embedder's [`AdaptiveRules::tolerates_partition_count_change`]
/// accepts it.
#[allow(deprecated)]
fn tolerates_partition_count_change(plan: &dyn ExecutionPlan) -> bool {
pub(crate) fn tolerates_partition_count_change(plan: &dyn ExecutionPlan) -> bool {
plan.is::<ProjectionExec>()
|| plan.is::<FilterExec>()
|| plan.is::<AggregateExec>()
Expand Down
Loading
Loading