From 46c809cdfdc9d60b6882741a9f685f1635e98b2f Mon Sep 17 00:00:00 2001 From: Xuezhao Liu Date: Fri, 23 Oct 2026 08:04:55 +0000 Subject: [PATCH] DAOS-19591 container: resend EC boundary to reintegrated targets After a target is reintegrated its EC aggregation boundary restarts from zero and the container shard stays paused until the leader pushes the boundary again. That push never comes in two cases: 1. The rank's report of a zero epoch is ignored by the leader because ds_cont_leader_update_track_eph() only ever raises the per-rank value, so the global minimum does not change and cont_agg_eph_sync() skips the unchanged boundary forever. 2. If a target (SSD) of a live engine was evicted and reintegrated, ds_cont_tgt_ec_eph_query_ult() sees min_ec_agg_eph == 0 with a non-zero last reported epoch and skips the report entirely, so the leader does not even see the zero. Leader side: take a zero report as the rank's current value, so that a zero minimum tells that some target waits for the boundary. While that is the case, re-push the boundary every 30 seconds even if it is unchanged, once rebuild is done and reclaim is enabled. The pushed boundary cannot regress since the minimum is still raised to the rdb epoch. Rank side: always report a zero minimum epoch so the leader learns that a target of this rank waits for the boundary. Also fix the false "Sluggish EC boundary reporting" warning (#19015 on master): only warn when the epoch gap is not larger than the period observed by this leader (since it started tracking the container, or since rebuild ended / reclaim was enabled), a gap inherited from rdb at leader step-up is not a real stall. Signed-off-by: Xuezhao Liu --- src/container/srv_container.c | 36 +++++++++++++++++++++++++++-------- src/container/srv_internal.h | 9 ++++++++- src/container/srv_target.c | 6 ++++-- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/container/srv_container.c b/src/container/srv_container.c index c36b468ca05..141aa52be92 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -1814,14 +1814,15 @@ cont_track_eph_leader_alloc(struct cont_svc *cont_svc, uuid_t cont_uuid, eph_ldr->cte_servers_num = rank_nr; eph_ldr->cte_current_ec_agg_eph = 0; eph_ldr->cte_rdb_ec_agg_eph = 0; + eph_ldr->cte_ec_agg_warn_slug_ts = daos_gettime_coarse(); for (i = 0; i < rank_nr; i++) { eph_ldr->cte_server_ephs[i].re_rank = doms[i].do_comp.co_rank; eph_ldr->cte_server_ephs[i].re_ec_agg_eph = 0; eph_ldr->cte_server_ephs[i].re_stable_eph = 0; - eph_ldr->cte_server_ephs[i].re_ec_agg_eph_update_ts = daos_gettime_coarse(); + eph_ldr->cte_server_ephs[i].re_ec_agg_eph_update_ts = + eph_ldr->cte_ec_agg_warn_slug_ts; } d_list_add(&eph_ldr->cte_list, &cont_svc->cs_cont_ephs_leader_list); - eph_ldr->cte_ec_agg_warn_slug_ts = daos_gettime_coarse(); *leader_p = eph_ldr; out: if (rc) { @@ -1897,7 +1898,11 @@ ds_cont_leader_update_track_eph(uuid_t pool_uuid, uuid_t cont_uuid, d_rank_t ran for (i = 0; i < eph_ldr->cte_servers_num; i++) { if (eph_ldr->cte_server_ephs[i].re_rank == rank) { - if (eph_ldr->cte_server_ephs[i].re_ec_agg_eph < ec_agg_eph) { + /* A zero report means a target of this rank was just (re)started + * and waits for the boundary, take it as is. + */ + if (eph_ldr->cte_server_ephs[i].re_ec_agg_eph < ec_agg_eph || + ec_agg_eph == 0) { eph_ldr->cte_server_ephs[i].re_ec_agg_eph = ec_agg_eph; eph_ldr->cte_server_ephs[i].re_ec_agg_eph_update_ts = daos_gettime_coarse(); @@ -2159,6 +2164,9 @@ cont_agg_eph_store(struct cont_svc *svc, uuid_t cont_uuid, uint64_t ec_agg_eph, return rc; } +#define TRACK_EPH_INTV (5ULL * 1000) /* ms interval to check */ +#define TRACK_EPH_RESEND_INTV 30 /* seconds between resends of unchanged boundary */ + static void cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) { @@ -2169,6 +2177,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) daos_epoch_t min_ec_agg_eph; daos_epoch_t min_stable_eph; uint64_t cur_ts; + bool resend; int i; int warn_slug_ranks = 8; /* 8 ranks at most */ int cont_num = 0; @@ -2207,8 +2216,12 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) min_ec_agg_eph = DAOS_EPOCH_MAX; min_stable_eph = DAOS_EPOCH_MAX; cur_ts = daos_gettime_coarse(); - if (ds_pool_is_rebuilding(pool) || pool->sp_reclaim == DAOS_RECLAIM_DISABLED) + if (ds_pool_is_rebuilding(pool) || pool->sp_reclaim == DAOS_RECLAIM_DISABLED) { eph_ldr->cte_ec_agg_warn_slug_ts = cur_ts; + resend = false; + } else { + resend = true; + } for (i = 0; i < eph_ldr->cte_servers_num; i++) { d_rank_t rank = eph_ldr->cte_server_ephs[i].re_rank; @@ -2235,13 +2248,20 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) min_stable_eph = eph_ldr->cte_server_ephs[i].re_stable_eph; } + /* A zero epoch means some (re)started target of that rank waits for the + * boundary. Periodically resend even an unchanged boundary to it, once + * rebuild is done and reclaim is enabled. + */ + resend = resend && min_ec_agg_eph == 0 && + cur_ts - eph_ldr->cte_refresh_ts >= TRACK_EPH_RESEND_INTV; + /* for reboot case the ea_rdb_eph possibly higher than min_eph */ if (min_ec_agg_eph < eph_ldr->cte_rdb_ec_agg_eph) min_ec_agg_eph = eph_ldr->cte_rdb_ec_agg_eph; if (min_ec_agg_eph == eph_ldr->cte_current_ec_agg_eph && min_stable_eph == eph_ldr->cte_current_stable_eph && - eph_ldr->cte_current_ec_agg_eph != 0) { + eph_ldr->cte_current_ec_agg_eph != 0 && !resend) { if (cont_num % 10 == 0) dss_sleep(0); continue; @@ -2261,8 +2281,8 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) cur_eph = d_hlc2sec(eph_ldr->cte_current_ec_agg_eph); new_eph = d_hlc2sec(min_ec_agg_eph); - if ((cur_ts > eph_ldr->cte_ec_agg_warn_slug_ts + 600) && cur_eph && - (new_eph > cur_eph) && (new_eph - cur_eph) >= 600) + if (cur_eph && (new_eph > cur_eph) && (new_eph - cur_eph) >= 600 && + (cur_ts - eph_ldr->cte_ec_agg_warn_slug_ts) >= (new_eph - cur_eph)) D_WARN(DF_CONT ": Sluggish EC boundary reporting. " "cur:" DF_U64 " new:" DF_U64 " gap:" DF_U64 "\n", DP_CONT(svc->cs_pool_uuid, eph_ldr->cte_cont_uuid), cur_eph, new_eph, @@ -2318,6 +2338,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) } eph_ldr->cte_current_ec_agg_eph = min_ec_agg_eph; eph_ldr->cte_current_stable_eph = min_stable_eph; + eph_ldr->cte_refresh_ts = daos_gettime_coarse(); } ABT_mutex_unlock(svc->cs_cont_ephs_mutex); @@ -2340,7 +2361,6 @@ ds_cont_svc_refresh_agg_eph(uuid_t pool_uuid) return 0; } -#define TRACK_EPH_INTV (5ULL * 1000) /* seconds interval to check*/ static void cont_track_eph_leader_ult(void *arg) { diff --git a/src/container/srv_internal.h b/src/container/srv_internal.h index 4f04ba018ee..6f850b4b778 100644 --- a/src/container/srv_internal.h +++ b/src/container/srv_internal.h @@ -75,7 +75,14 @@ struct cont_track_eph_leader { d_list_t cte_list; int cte_servers_num; uint32_t cte_deleted : 1; - /* TS to check for ec_agg_eph sluggish warning */ + /* TS of the last successful boundary push to the ranks */ + uint64_t cte_refresh_ts; + /* Start of the period in which EC boundary progress is accounted for sluggish + * warnings: set when this leader starts tracking the container, moved forward + * while rebuilding or reclaim is disabled. A boundary jump larger than this + * period was not observed by this leader (it was inherited from rdb at leader + * step-up or made during rebuild), so it is not a real stall. + */ uint64_t cte_ec_agg_warn_slug_ts; }; diff --git a/src/container/srv_target.c b/src/container/srv_target.c index 7e9b13f804a..68e5a9ba615 100644 --- a/src/container/srv_target.c +++ b/src/container/srv_target.c @@ -2839,10 +2839,12 @@ ds_cont_eph_report(struct ds_pool *pool) } /* if aggregation enabled, make sure to report ec_agg_eph at the start phase - * when min_ec_agg_eph and cte_last_ec_agg_epoch are both zero. + * when min_ec_agg_eph and cte_last_ec_agg_epoch are both zero. Also report a + * zero min_ec_agg_eph after a target of this rank is reintegrated, so that the + * leader knows some target waits for the boundary. */ if (min_ec_agg_eph == DAOS_EPOCH_MAX || min_stable_eph == DAOS_EPOCH_MAX || - (ec_eph->cte_last_ec_agg_epoch != 0 && + (min_ec_agg_eph != 0 && ec_eph->cte_last_ec_agg_epoch != 0 && min_ec_agg_eph <= ec_eph->cte_last_ec_agg_epoch && min_stable_eph <= ec_eph->cte_last_stable_epoch)) { if (min_ec_agg_eph > 0 && min_stable_eph > 0 &&