From c00065d7481a752713956f5bbd48128b0be94f6b Mon Sep 17 00:00:00 2001 From: Xuezhao Liu Date: Mon, 19 Oct 2026 16:38:36 +0000 Subject: [PATCH 1/2] DAOS-19591 container: recover EC boundaries after reintegration Reintegrated or extended targets can miss the last EC aggregation boundary and wait indefinitely for a refresh. Their zero epochs may be suppressed by the rank's previous report, or they may have no local VOS container and therefore never report. Refreshing only when the minimum epoch changes can leave the container's boundary and space reclamation stuck. Track such waiting ranks with ea_need_refresh, set when a rank's zero report is suppressed by its earlier report or when some non-failed rank has reported nothing yet. While it is set, and rebuild is inactive and reclaim is enabled, resend the unchanged boundary every 30 seconds; the flag is cleared before each push so steady-state traffic drops back to zero. Allow leader pushes to create missing VOS containers, using deep-stack thread collectives and tolerating concurrent creation. Mark pushes using the otherwise unused boundary payload rank; clear that marker on fetch copies so cached boundaries cannot recreate containers. The IV wire layout remains unchanged. When destroying a container, wait up to 10 seconds for an in-flight boundary push and return DER_BUSY if it has not finished, rather than proceeding after a timeout. Stop tracking before the destroy broadcast, including orphan cleanup, and check retired tracking entries as well. Skip refreshes after RDB errors or deletion during an RDB operation, and retire entries for nonexistent containers. Also suppress sluggish-boundary warnings for epoch jumps larger than the current leader's tracking age. Features: container rebuild Signed-off-by: Xuezhao Liu Signed-off-by: Mohamad Chaarawi --- src/container/container_iv.c | 16 ++-- src/container/srv_container.c | 137 ++++++++++++++++++++++++---------- src/container/srv_internal.h | 15 +++- src/container/srv_target.c | 10 ++- 4 files changed, 129 insertions(+), 49 deletions(-) diff --git a/src/container/container_iv.c b/src/container/container_iv.c index b61bd28e59a..aea055c0a27 100644 --- a/src/container/container_iv.c +++ b/src/container/container_iv.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2019-2023 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -194,6 +194,10 @@ cont_iv_ent_copy(struct ds_iv_entry *entry, struct cont_iv_key *key, break; case IV_CONT_AGG_EPOCH_BOUNDRY: dst->iv_agg_eph.eph = src->iv_agg_eph.eph; + /* Fetches are not tracked by ea_refreshing: do not propagate the push's + * creation marker, or a late fetch could recreate a destroyed container. + */ + dst->iv_agg_eph.rank = 0; break; default: rc = -DER_INVAL; @@ -571,7 +575,7 @@ cont_iv_ent_fetch(struct ds_iv_entry *entry, struct ds_iv_key *key, */ rc = ds_cont_tgt_refresh_agg_eph(entry->ns->iv_pool_uuid, civ_key->cont_uuid, - ec_agg_eph); + ec_agg_eph, false); if (rc == 0) goto again; } @@ -622,9 +626,9 @@ cont_iv_ent_agg_eph_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key, struct cont_iv_key *civ_key = key2priv(key); int rc; - rc = ds_cont_tgt_refresh_agg_eph(entry->ns->iv_pool_uuid, - civ_key->cont_uuid, - civ_ent->iv_agg_eph.eph); + rc = ds_cont_tgt_refresh_agg_eph(entry->ns->iv_pool_uuid, civ_key->cont_uuid, + civ_ent->iv_agg_eph.eph, + civ_ent->iv_agg_eph.rank == CRT_NO_RANK); return rc; } @@ -1144,6 +1148,8 @@ cont_iv_ec_agg_eph_update_internal(void *ns, uuid_t cont_uuid, DP_UUID(cont_uuid), op, DP_RC(rc)); return rc; } + if (op == IV_CONT_AGG_EPOCH_BOUNDRY) + iv_entry.iv_agg_eph.rank = CRT_NO_RANK; rc = cont_iv_update(ns, op, cont_uuid, &iv_entry, sizeof(iv_entry), shortcut, sync_mode, false /* retry */); diff --git a/src/container/srv_container.c b/src/container/srv_container.c index bf9ebdeddaa..1909f05e2b1 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -1541,7 +1541,7 @@ evict_hdls(struct rdb_tx *tx, struct cont *cont, bool force, struct ds_pool_hdl return rc; } -static void +static int cont_ec_agg_delete(struct cont_svc *svc, uuid_t cont_uuid); static int @@ -1591,28 +1591,30 @@ cont_destroy(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont, if (rc != 0) goto out_prop; + rc = cont_ec_agg_delete(cont->c_svc, cont->c_uuid); + if (rc != 0) + goto out_prop; + rc = cont_destroy_bcast(rpc->cr_ctx, cont->c_svc, cont->c_uuid); if (rc != 0) goto out_prop; - cont_ec_agg_delete(cont->c_svc, cont->c_uuid); - - if (pool_hdl->sph_global_ver >= DAOS_POOL_GLOBAL_VERSION_WITH_OIT_OID_KVS) { - need_destroy_oid_oit_kvs = true; - } else { - d_iov_t value; - - d_iov_set(&value, NULL, 0); - rc = rdb_tx_lookup(tx, &cont->c_prop, &ds_cont_prop_oit_oids, &value); - if (rc && rc != -DER_NONEXIST) { - DL_ERROR(rc, "failed to lookup oit oid kvs pool/cont: " DF_CONTF, - DP_CONT(pool_hdl->sph_pool->sp_uuid, cont->c_uuid)); - goto out_prop; - } - /* There was a bug that oit oids might be created already see DAOS-14799 */ + if (pool_hdl->sph_global_ver >= DAOS_POOL_GLOBAL_VERSION_WITH_OIT_OID_KVS) { + need_destroy_oid_oit_kvs = true; + } else { + d_iov_t value; + + d_iov_set(&value, NULL, 0); + rc = rdb_tx_lookup(tx, &cont->c_prop, &ds_cont_prop_oit_oids, &value); + if (rc && rc != -DER_NONEXIST) { + DL_ERROR(rc, "failed to lookup oit oid kvs pool/cont: " DF_CONTF, + DP_CONT(pool_hdl->sph_pool->sp_uuid, cont->c_uuid)); + goto out_prop; + } + /* There was a bug that oit oids might be created already see DAOS-14799 */ if (rc == 0) need_destroy_oid_oit_kvs = true; - } + } /* Destroy oit oids index KVS. */ if (need_destroy_oid_oit_kvs) { @@ -1712,10 +1714,12 @@ cont_ec_agg_alloc(struct cont_svc *cont_svc, uuid_t cont_uuid, ec_agg->ea_servers_num = rank_nr; ec_agg->ea_current_eph = 0; ec_agg->ea_rdb_eph = 0; + ec_agg->ea_start_ts = daos_gettime_coarse(); + ec_agg->ea_warn_slug_ts = ec_agg->ea_start_ts; for (i = 0; i < rank_nr; i++) { ec_agg->ea_server_ephs[i].rank = doms[i].do_comp.co_rank; ec_agg->ea_server_ephs[i].eph = 0; - ec_agg->ea_server_ephs[i].ee_update_ts = daos_gettime_coarse(); + ec_agg->ea_server_ephs[i].ee_update_ts = ec_agg->ea_start_ts; } d_list_add(&ec_agg->ea_list, &cont_svc->cs_ec_agg_list); *ec_aggp = ec_agg; @@ -1729,19 +1733,35 @@ cont_ec_agg_alloc(struct cont_svc *cont_svc, uuid_t cont_uuid, return rc; } -static void +static int cont_ec_agg_delete(struct cont_svc *svc, uuid_t cont_uuid) { - struct cont_ec_agg *ec_agg; - - ec_agg = cont_ec_agg_lookup(svc, cont_uuid); - if (ec_agg == NULL) - return; + struct cont_ec_agg *ec_agg; + int retry = 0; - /* Set ea_deleted flag to destroy it inside cont_agg_eph_leader_ult() - * to avoid list iteration broken. + /* A boundary push may create VOS containers. Do not destroy until it completes, + * but bound the wait (cs_lock may held) in case the push keeps retrying on an + * unreachable rank. */ - ec_agg->ea_deleted = 1; +again: + d_list_for_each_entry(ec_agg, &svc->cs_ec_agg_list, ea_list) { + if (uuid_compare(ec_agg->ea_cont_uuid, cont_uuid) == 0 && ec_agg->ea_refreshing) { + if (retry++ < 30) { + dss_sleep(300 /* ms, 10s in total */); + goto again; + } + D_DEBUG(DB_MD, DF_CONT ": EC boundary refresh in flight\n", + DP_CONT(svc->cs_pool_uuid, cont_uuid)); + return -DER_BUSY; + } + } + + /* Defer freeing to cont_agg_eph_sync() to preserve its list iteration. */ + d_list_for_each_entry(ec_agg, &svc->cs_ec_agg_list, ea_list) { + if (uuid_compare(ec_agg->ea_cont_uuid, cont_uuid) == 0) + ec_agg->ea_deleted = 1; + } + return 0; } /** @@ -1776,6 +1796,9 @@ ds_cont_leader_update_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, if (ec_agg->ea_server_ephs[i].eph < eph) { ec_agg->ea_server_ephs[i].eph = eph; ec_agg->ea_server_ephs[i].ee_update_ts = daos_gettime_coarse(); + } else if (eph == 0 && ec_agg->ea_current_eph != 0) { + /* A recovered target of this rank waits for the boundary. */ + ec_agg->ea_need_refresh = 1; } break; } @@ -1808,6 +1831,7 @@ struct refresh_vos_agg_eph_arg { uuid_t pool_uuid; uuid_t cont_uuid; daos_epoch_t min_eph; + bool create; }; int @@ -1818,9 +1842,19 @@ cont_refresh_vos_agg_eph_one(void *data) int rc; rc = ds_cont_child_lookup(arg->pool_uuid, arg->cont_uuid, &cont_child); - if (rc) { - DL_CDEBUG(rc != 0 && rc != -DER_SHUTDOWN, DLOG_ERR, DB_MD, rc, - DF_CONT " lookup cont failed", DP_CONT(arg->pool_uuid, arg->cont_uuid)); + if (rc == -DER_NONEXIST && arg->create) { + /* Reintegration may not migrate any records for this container. Only a + * leader push, serialized with destroy, may create the missing container. + */ + rc = ds_cont_child_open_create(arg->pool_uuid, arg->cont_uuid, &cont_child); + DL_CDEBUG(rc != 0 && rc != -DER_SHUTDOWN, DLOG_ERR, DLOG_INFO, rc, + DF_CONT " create missing vos container", + DP_CONT(arg->pool_uuid, arg->cont_uuid)); + if (rc != 0) + return rc; + } else if (rc != 0) { + DL_CDEBUG(rc != -DER_SHUTDOWN, DLOG_ERR, DB_MD, rc, DF_CONT " lookup cont failed", + DP_CONT(arg->pool_uuid, arg->cont_uuid)); return rc; } @@ -1838,8 +1872,7 @@ cont_refresh_vos_agg_eph_one(void *data) } int -ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, - daos_epoch_t eph) +ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, daos_epoch_t eph, bool create) { struct refresh_vos_agg_eph_arg arg; static uint64_t cnt; @@ -1849,10 +1882,12 @@ ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, uuid_copy(arg.pool_uuid, pool_uuid); uuid_copy(arg.cont_uuid, cont_uuid); arg.min_eph = eph; + arg.create = create; - rc = ds_pool_task_collective(pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN | - PO_COMP_ST_DOWNOUT, cont_refresh_vos_agg_eph_one, - &arg, DSS_ULT_FL_PERIODIC); + /* VOS container creation may yield and needs a deep stack. */ + rc = ds_pool_thread_collective( + pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN | PO_COMP_ST_DOWNOUT, + cont_refresh_vos_agg_eph_one, &arg, DSS_ULT_DEEP_STACK | DSS_ULT_FL_PERIODIC); if (rc) { DL_ERROR(rc, DF_CONT ": refresh ec_agg_eph " DF_X64 " failed.", DP_CONT(pool_uuid, cont_uuid), eph); @@ -1996,6 +2031,7 @@ cont_agg_eph_store(struct cont_svc *svc, uuid_t cont_uuid, uint64_t ec_agg_eph, return rc; } +#define EC_AGG_EPH_INTV (10ULL * 1000) /* seconds interval to check*/ static void cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) { @@ -2030,12 +2066,15 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) rc = cont_agg_eph_load(svc, ec_agg->ea_cont_uuid, &ec_agg->ea_rdb_eph); if (rc) { if (rc == -DER_NONEXIST) { + /* destroyed, free the entry in the next round */ + ec_agg->ea_deleted = 1; DL_INFO(rc, DF_CONT " container skipped", DP_CONT(svc->cs_pool_uuid, ec_agg->ea_cont_uuid)); continue; } DL_ERROR(rc, DF_CONT ": cont_agg_eph_load failed.", DP_CONT(svc->cs_pool_uuid, ec_agg->ea_cont_uuid)); + continue; } } @@ -2070,11 +2109,21 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) min_eph = ec_agg->ea_server_ephs[i].eph; } + /* some rank reported nothing, it may lack the VOS container */ + if (min_eph == 0) + ec_agg->ea_need_refresh = 1; + /* for reboot case the ea_rdb_eph possibly higher than min_eph */ if (min_eph < ec_agg->ea_rdb_eph) min_eph = ec_agg->ea_rdb_eph; - if (min_eph == ec_agg->ea_current_eph && ec_agg->ea_current_eph != 0) + /* A recovered target may report nothing, or its zero epoch may be suppressed + * by the rank's last report. Periodically resend even an unchanged boundary + * while some rank waits for it, once rebuild is done and reclaim is enabled. + */ + if (min_eph == ec_agg->ea_current_eph && ec_agg->ea_current_eph != 0 && + (!warn_sluggish || !ec_agg->ea_need_refresh || + cur_ts - ec_agg->ea_refresh_ts < EC_AGG_EPH_INTV * 3 / 1000)) continue; /** @@ -2094,6 +2143,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) ec_agg->ea_warn_slug_ts = cur_ts; } else if (cur_eph && new_eph > cur_eph && (new_eph - cur_eph) >= 600 && + (cur_ts - ec_agg->ea_start_ts) >= (new_eph - cur_eph) && (cur_ts - ec_agg->ea_warn_slug_ts) >= 600) { ec_agg->ea_warn_slug_ts = cur_ts; D_WARN(DF_CONT ": Sluggish EC boundary reporting. " @@ -2110,6 +2160,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) &ec_agg->ea_rdb_eph); if (rc) { if (rc == -DER_NONEXIST) { + ec_agg->ea_deleted = 1; DL_INFO(rc, DF_CONT " container skipped", DP_CONT(svc->cs_pool_uuid, ec_agg->ea_cont_uuid)); continue; @@ -2117,11 +2168,19 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) DL_ERROR(rc, DF_CONT ": rdb_tx_update ec_agg_eph " DF_X64 " failed.", DP_CONT(svc->cs_pool_uuid, ec_agg->ea_cont_uuid), min_eph); + continue; } } + if (ec_agg->ea_deleted) + continue; + + /* clear before the push (which yields), a request arriving meanwhile is kept */ + ec_agg->ea_need_refresh = 0; + ec_agg->ea_refreshing = 1; rc = cont_iv_ec_agg_eph_refresh(pool->sp_iv_ns, ec_agg->ea_cont_uuid, min_eph, svc->cs_ec_leader_ephs_req); + ec_agg->ea_refreshing = 0; if (rc) { DL_CDEBUG(rc == -DER_NONEXIST, DLOG_INFO, DLOG_ERR, rc, DF_CONT ": refresh failed", @@ -2144,6 +2203,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc) continue; } ec_agg->ea_current_eph = min_eph; + ec_agg->ea_refresh_ts = daos_gettime_coarse(); } ABT_mutex_unlock(svc->cs_ec_agg_mutex); @@ -2166,7 +2226,6 @@ ds_cont_svc_refresh_agg_eph(uuid_t pool_uuid) return 0; } -#define EC_AGG_EPH_INTV (10ULL * 1000) /* seconds interval to check*/ static void cont_agg_eph_leader_ult(void *arg) { @@ -6278,9 +6337,9 @@ ds_cont_destroy_orphan(struct cont_svc *svc, uuid_t uuid) /* Forbid to destroy non-orphan container. */ D_GOTO(out, rc = -DER_BUSY); - rc = cont_destroy_bcast(dss_get_module_info()->dmi_ctx, svc, uuid); + rc = cont_ec_agg_delete(svc, uuid); if (rc == 0) - cont_ec_agg_delete(svc, uuid); + rc = cont_destroy_bcast(dss_get_module_info()->dmi_ctx, svc, uuid); out: D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, diff --git a/src/container/srv_internal.h b/src/container/srv_internal.h index d53205a4be1..42998c67a68 100644 --- a/src/container/srv_internal.h +++ b/src/container/srv_internal.h @@ -69,11 +69,16 @@ struct cont_ec_agg { uuid_t ea_cont_uuid; daos_epoch_t ea_current_eph; daos_epoch_t ea_rdb_eph; + /* Ignore boundary jumps predating this leader's tracking. */ + uint64_t ea_start_ts; + uint64_t ea_refresh_ts; uint64_t ea_warn_slug_ts; struct ec_eph *ea_server_ephs; d_list_t ea_list; int ea_servers_num; - uint32_t ea_deleted : 1; + uint32_t ea_deleted : 1, ea_refreshing : 1, + /* some rank waits for the boundary (reported 0 or nothing), see cont_agg_eph_sync() */ + ea_need_refresh : 1; }; /* @@ -165,6 +170,10 @@ struct cont_iv_prop { struct cont_iv_agg_eph { daos_epoch_t eph; + /* Reports carry the reporter rank. For boundaries, CRT_NO_RANK allows a push to create + * missing VOS containers; fetch replies use 0 to disallow creation, not as + * a reporter rank. Only leader pushes are tracked against container destroy. + */ d_rank_t rank; }; @@ -275,8 +284,8 @@ int ds_cont_tgt_snapshots_update(uuid_t pool_uuid, uuid_t cont_uuid, uint64_t *snapshots, int snap_count); int ds_cont_tgt_snapshots_refresh(uuid_t pool_uuid, uuid_t cont_uuid); int ds_cont_tgt_close(uuid_t pool_uuid, uuid_t cont_hdl_uuid); -int ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, - daos_epoch_t eph); +int + ds_cont_tgt_refresh_agg_eph(uuid_t pool_uuid, uuid_t cont_uuid, daos_epoch_t eph, bool create); int ds_cont_tgt_prop_update(uuid_t pool_uuid, uuid_t cont_uuid, daos_prop_t *prop); /* oid_iv.c */ diff --git a/src/container/srv_target.c b/src/container/srv_target.c index ef05ad89e3b..244ad979a88 100644 --- a/src/container/srv_target.c +++ b/src/container/srv_target.c @@ -1428,6 +1428,7 @@ cont_child_create_start(uuid_t pool_uuid, uuid_t cont_uuid, uint32_t pm_ver, bool *started, struct ds_cont_child **cont_out) { struct ds_pool_child *pool_child; + bool created = true; int rc; pool_child = ds_pool_child_lookup(pool_uuid); @@ -1451,11 +1452,16 @@ cont_child_create_start(uuid_t pool_uuid, uuid_t cont_uuid, uint32_t pm_ver, DP_CONT(pool_uuid, cont_uuid)); rc = vos_cont_create(pool_child->spc_hdl, cont_uuid); + if (unlikely(rc == -DER_EXIST)) { + /* Creation may yield; do not roll back another sponsor's container. */ + rc = 0; + created = false; + } if (!rc) { rc = cont_child_start(pool_child, cont_uuid, started, cont_out); if (rc == 0) { (*cont_out)->sc_status_pm_ver = pm_ver; - } else { + } else if (created) { int rc_tmp; rc_tmp = vos_cont_destroy(pool_child->spc_hdl, cont_uuid); @@ -1466,7 +1472,7 @@ cont_child_create_start(uuid_t pool_uuid, uuid_t cont_uuid, uint32_t pm_ver, } ds_pool_child_put(pool_child); - return rc == 0 ? 1 : rc; + return rc == 0 ? (created ? 1 : 0) : rc; } int From 78ae37c8ee898d6bc534da20f77c0da1a8141ffd Mon Sep 17 00:00:00 2001 From: Xuezhao Liu Date: Tue, 20 Oct 2026 13:11:26 +0000 Subject: [PATCH 2/2] DAOS-19591 container: report zero EC agg epoch after target reintegration When a target is excluded and reintegrated on a running engine (for example an evicted SSD), its container shards start with no EC aggregation boundary and wait for the leader to resend it. The rank's reporting ULT however skipped a zero minimum once it had reported a non-zero epoch before, so the leader never learned that a shard was waiting, kept the rank's stale epoch and pinned the pool-wide boundary. Report a zero minimum even after earlier non-zero reports. The leader ignores the value itself but marks the container for a boundary resend, which validates the boundary on the reintegrated target and lets it aggregate and report again. Signed-off-by: Xuezhao Liu --- src/container/srv_target.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/container/srv_target.c b/src/container/srv_target.c index 244ad979a88..598bded6da9 100644 --- a/src/container/srv_target.c +++ b/src/container/srv_target.c @@ -2569,10 +2569,13 @@ ds_cont_tgt_ec_eph_query_ult(void *data) continue; /* if aggregation enabled, make sure to report ec_agg_eph at the start phase - * when min_eph and ce_last_eph are both zero. + * when min_eph and ce_last_eph are both zero. Also report min_eph 0 after + * non-zero reports: a reintegrated target has no boundary yet and waits + * for the leader to resend it, see ds_cont_leader_update_agg_eph(). */ if (min_eph == DAOS_EPOCH_MAX || - (ec_eph->ce_last_eph != 0 && min_eph <= ec_eph->ce_last_eph)) { + (ec_eph->ce_last_eph != 0 && min_eph != 0 && + min_eph <= ec_eph->ce_last_eph)) { if (min_eph > 0 && min_eph < ec_eph->ce_last_eph) D_ERROR("ignore for now "DF_X64" < "DF_X64 " "DF_UUID"\n", min_eph, ec_eph->ce_last_eph,