Skip to content

DAOS-19591 container: resend EC boundary to reintegrated targets #19356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
liuxuezhao wants to merge 1 commit into
base: release/2.8
Choose a base branch
from
Open

DAOS-19591 container: resend EC boundary to reintegrated targets #19356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions src/container/srv_container.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1814,14 +1814,15 @@ cont_track_eph_leader_alloc(struct cont_svc *cont_svc, uuid_t cont_uuid,
eph_ldr->cte_servers_num = rank_nr;
eph_ldr->cte_current_ec_agg_eph = 0;
eph_ldr->cte_rdb_ec_agg_eph = 0;
eph_ldr->cte_ec_agg_warn_slug_ts = daos_gettime_coarse();
for (i = 0; i < rank_nr; i++) {
eph_ldr->cte_server_ephs[i].re_rank = doms[i].do_comp.co_rank;
eph_ldr->cte_server_ephs[i].re_ec_agg_eph = 0;
eph_ldr->cte_server_ephs[i].re_stable_eph = 0;
eph_ldr->cte_server_ephs[i].re_ec_agg_eph_update_ts = daos_gettime_coarse();
eph_ldr->cte_server_ephs[i].re_ec_agg_eph_update_ts =
eph_ldr->cte_ec_agg_warn_slug_ts;
}
d_list_add(&eph_ldr->cte_list, &cont_svc->cs_cont_ephs_leader_list);
eph_ldr->cte_ec_agg_warn_slug_ts = daos_gettime_coarse();
*leader_p = eph_ldr;
out:
if (rc) {
Expand Down Expand Up @@ -1897,7 +1898,11 @@ ds_cont_leader_update_track_eph(uuid_t pool_uuid, uuid_t cont_uuid, d_rank_t ran

for (i = 0; i < eph_ldr->cte_servers_num; i++) {
if (eph_ldr->cte_server_ephs[i].re_rank == rank) {
if (eph_ldr->cte_server_ephs[i].re_ec_agg_eph < ec_agg_eph) {
/* A zero report means a target of this rank was just (re)started
* and waits for the boundary, take it as is.
*/
if (eph_ldr->cte_server_ephs[i].re_ec_agg_eph < ec_agg_eph ||
ec_agg_eph == 0) {
eph_ldr->cte_server_ephs[i].re_ec_agg_eph = ec_agg_eph;
eph_ldr->cte_server_ephs[i].re_ec_agg_eph_update_ts =
daos_gettime_coarse();
Expand Down Expand Up @@ -2159,6 +2164,9 @@ cont_agg_eph_store(struct cont_svc *svc, uuid_t cont_uuid, uint64_t ec_agg_eph,
return rc;
}

#define TRACK_EPH_INTV (5ULL * 1000) /* ms interval to check */
#define TRACK_EPH_RESEND_INTV 30 /* seconds between resends of unchanged boundary */

static void
cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc)
{
Expand All @@ -2169,6 +2177,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc)
daos_epoch_t min_ec_agg_eph;
daos_epoch_t min_stable_eph;
uint64_t cur_ts;
bool resend;
int i;
int warn_slug_ranks = 8; /* 8 ranks at most */
int cont_num = 0;
Expand Down Expand Up @@ -2207,8 +2216,12 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc)
min_ec_agg_eph = DAOS_EPOCH_MAX;
min_stable_eph = DAOS_EPOCH_MAX;
cur_ts = daos_gettime_coarse();
if (ds_pool_is_rebuilding(pool) || pool->sp_reclaim == DAOS_RECLAIM_DISABLED)
if (ds_pool_is_rebuilding(pool) || pool->sp_reclaim == DAOS_RECLAIM_DISABLED) {
eph_ldr->cte_ec_agg_warn_slug_ts = cur_ts;
resend = false;
} else {
resend = true;
}
for (i = 0; i < eph_ldr->cte_servers_num; i++) {
d_rank_t rank = eph_ldr->cte_server_ephs[i].re_rank;

Expand All @@ -2235,13 +2248,20 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc)
min_stable_eph = eph_ldr->cte_server_ephs[i].re_stable_eph;
}

/* A zero epoch means some (re)started target of that rank waits for the
* boundary. Periodically resend even an unchanged boundary to it, once
* rebuild is done and reclaim is enabled.
*/
resend = resend && min_ec_agg_eph == 0 &&
cur_ts - eph_ldr->cte_refresh_ts >= TRACK_EPH_RESEND_INTV;

/* for reboot case the ea_rdb_eph possibly higher than min_eph */
if (min_ec_agg_eph < eph_ldr->cte_rdb_ec_agg_eph)
min_ec_agg_eph = eph_ldr->cte_rdb_ec_agg_eph;

if (min_ec_agg_eph == eph_ldr->cte_current_ec_agg_eph &&
min_stable_eph == eph_ldr->cte_current_stable_eph &&
eph_ldr->cte_current_ec_agg_eph != 0) {
eph_ldr->cte_current_ec_agg_eph != 0 && !resend) {
if (cont_num % 10 == 0)
dss_sleep(0);
continue;
Expand All @@ -2261,8 +2281,8 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc)

cur_eph = d_hlc2sec(eph_ldr->cte_current_ec_agg_eph);
new_eph = d_hlc2sec(min_ec_agg_eph);
if ((cur_ts > eph_ldr->cte_ec_agg_warn_slug_ts + 600) && cur_eph &&
(new_eph > cur_eph) && (new_eph - cur_eph) >= 600)
if (cur_eph && (new_eph > cur_eph) && (new_eph - cur_eph) >= 600 &&
(cur_ts - eph_ldr->cte_ec_agg_warn_slug_ts) >= (new_eph - cur_eph))
D_WARN(DF_CONT ": Sluggish EC boundary reporting. "
"cur:" DF_U64 " new:" DF_U64 " gap:" DF_U64 "\n",
DP_CONT(svc->cs_pool_uuid, eph_ldr->cte_cont_uuid), cur_eph, new_eph,
Expand Down Expand Up @@ -2318,6 +2338,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc)
}
eph_ldr->cte_current_ec_agg_eph = min_ec_agg_eph;
eph_ldr->cte_current_stable_eph = min_stable_eph;
eph_ldr->cte_refresh_ts = daos_gettime_coarse();
}
ABT_mutex_unlock(svc->cs_cont_ephs_mutex);

Expand All @@ -2340,7 +2361,6 @@ ds_cont_svc_refresh_agg_eph(uuid_t pool_uuid)
return 0;
}

#define TRACK_EPH_INTV (5ULL * 1000) /* seconds interval to check*/
static void
cont_track_eph_leader_ult(void *arg)
{
Expand Down
9 changes: 8 additions & 1 deletion src/container/srv_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,14 @@ struct cont_track_eph_leader {
d_list_t cte_list;
int cte_servers_num;
uint32_t cte_deleted : 1;
/* TS to check for ec_agg_eph sluggish warning */
/* TS of the last successful boundary push to the ranks */
uint64_t cte_refresh_ts;
/* Start of the period in which EC boundary progress is accounted for sluggish
* warnings: set when this leader starts tracking the container, moved forward
* while rebuilding or reclaim is disabled. A boundary jump larger than this
* period was not observed by this leader (it was inherited from rdb at leader
* step-up or made during rebuild), so it is not a real stall.
*/
uint64_t cte_ec_agg_warn_slug_ts;
};

Expand Down
6 changes: 4 additions & 2 deletions src/container/srv_target.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2839,10 +2839,12 @@ ds_cont_eph_report(struct ds_pool *pool)
}

/* if aggregation enabled, make sure to report ec_agg_eph at the start phase
* when min_ec_agg_eph and cte_last_ec_agg_epoch are both zero.
* when min_ec_agg_eph and cte_last_ec_agg_epoch are both zero. Also report a
* zero min_ec_agg_eph after a target of this rank is reintegrated, so that the
* leader knows some target waits for the boundary.
*/
if (min_ec_agg_eph == DAOS_EPOCH_MAX || min_stable_eph == DAOS_EPOCH_MAX ||
(ec_eph->cte_last_ec_agg_epoch != 0 &&
(min_ec_agg_eph != 0 && ec_eph->cte_last_ec_agg_epoch != 0 &&
min_ec_agg_eph <= ec_eph->cte_last_ec_agg_epoch &&
min_stable_eph <= ec_eph->cte_last_stable_epoch)) {
if (min_ec_agg_eph > 0 && min_stable_eph > 0 &&
Expand Down
Loading