From ca288c174a94b6a85013957903392d6c23d30521 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Fri, 18 Sep 2026 00:40:05 -0700 Subject: [PATCH] feat(kubernetes): retry Job Pods lost to infrastructure disruptions Add a Pod failure policy with a single Ignore rule on the DisruptionTarget condition. Pods the cluster takes away -- preemption, eviction, taint-based deletion, node loss, graceful node shutdown -- are no longer charged to the per-index budget, so the Job controller creates a replacement Pod. Exit codes are never read, so a failing task is still failed on its first attempt: backoff_limit_per_index stays 0. https://kubernetes.io/docs/tasks/job/pod-failure-policy/ --- .../launchers/kubernetes_launchers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 9b1a661..c0d282e 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -1218,6 +1218,25 @@ def launch_container_task( max_failed_indexes=0, completions=num_nodes, parallelism=num_nodes, + # Retry infrastructure disruptions only: + # DisruptionTarget -> Ignore -> not charged -> replacement Pod + # anything else -> charged -> budget is 0 -> index fails + # Kubernetes sets DisruptionTarget for preemption, eviction, taint-based + # deletion, node loss and graceful node shutdown -- never for a task's + # own exit code. + pod_failure_policy=k8s_client_lib.V1PodFailurePolicy( + rules=[ + k8s_client_lib.V1PodFailurePolicyRule( + action="Ignore", + on_pod_conditions=[ + k8s_client_lib.V1PodFailurePolicyOnPodConditionsPattern( + type="DisruptionTarget", + status="True", + ) + ], + ) + ] + ), ), )