From fae8bd5ed7c62c6fca628b72b2326bb761679a8a Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Thu, 4 Sep 2025 12:58:46 -0400 Subject: [PATCH 1/4] DAOS-17281 rebuild: handle admin stop during op:Fail_reclaim Before this change, upon a failed rebuild, both an op:Fail_reclaim and a retry of the original op:Rebuild were scheduled. With this change, the scheduling of the retry of the original op:Rebuild is deferred until the subsequent completion of op:Fail_reclaim. This is done so that if a rebuild stop command happens to arrive during Fail_reclaim, the administrator can stop an endless retry loop of op:Rebuild / Fail_reclaim from otherwise occurring. ds_rebuild_scheudle() now accepts additional arguments specifically interpreted when scheduling op:Fail_reclaim. So that when Fail_reclaim later completes and might schedule a retry of the original op:Rebuild, it has the appropriate pool map version and rebuild opcode to supply to that later ds_rebuild_schedule() call. rebuild_task_complete_schedule() is updated to schedule Fail_reclaim, and detect on its completion if a stop command was received, and decide whether to schedule a retry of original op:Rebuild. Exercise the new stop handling during Fail_reclaim from daos_test -v (simple rebuild, rebuild_many_objects_with_failure test) when run with --rebuild_interactive. This is done via a new fault injection, DAOS_REBUILD_ADMIN_STOP_RECLAIM. Features: rebuild Allow-unstable-test: true Signed-off-by: Kenneth Cain --- src/include/daos/common.h | 1 + src/include/daos_srv/rebuild.h | 3 +- src/pool/srv_pool.c | 4 +- src/rebuild/rebuild_internal.h | 7 +- src/rebuild/srv.c | 209 +++++++++++++++++++------- src/tests/suite/daos_rebuild_common.c | 25 +++ src/tests/suite/daos_rebuild_simple.c | 52 +++++-- src/tests/suite/daos_test.h | 2 + 8 files changed, 232 insertions(+), 71 deletions(-) diff --git a/src/include/daos/common.h b/src/include/daos/common.h index ba8dfc04667..7999879193a 100644 --- a/src/include/daos/common.h +++ b/src/include/daos/common.h @@ -822,6 +822,7 @@ enum { */ #define DAOS_FAIL_PARITY_EPOCH_DIFF (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x29) #define DAOS_FAIL_SHARD_NONEXIST (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x2a) +#define DAOS_REBUILD_ADMIN_STOP_RECLAIM (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x2b) #define DAOS_DTX_COMMIT_SYNC (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x30) #define DAOS_DTX_LEADER_ERROR (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x31) diff --git a/src/include/daos_srv/rebuild.h b/src/include/daos_srv/rebuild.h index e3f1d8a4794..0965f1f3b4a 100644 --- a/src/include/daos_srv/rebuild.h +++ b/src/include/daos_srv/rebuild.h @@ -86,7 +86,8 @@ typedef enum { int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t stable_eph, uint32_t layout_version, struct pool_target_id_list *tgts, - daos_rebuild_opc_t rebuild_op, bool stop_admin, uint64_t delay_sec); + daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op, + uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec); void ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank); int ds_rebuild_query(uuid_t pool_uuid, struct daos_rebuild_status *status); diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 40b6e0cc127..182d165fbd0 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -6238,6 +6238,7 @@ pool_check_upgrade_object_layout(struct rdb_tx *tx, struct pool_svc *svc, if (current_layout_ver < DS_POOL_OBJ_VERSION) { rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, upgrade_eph, DS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE, + RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */, false /* stop_admin */, 0); if (rc == 0) *scheduled_layout_upgrade = true; @@ -7554,7 +7555,8 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank, if (tgt_map_ver != 0) { rc = ds_rebuild_schedule(svc->ps_pool, tgt_map_ver, rebuild_eph, 0, &target_list, - RB_OP_REBUILD, false /* stop_admin */, delay); + RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, delay); if (rc != 0) { D_ERROR("rebuild fails rc: "DF_RC"\n", DP_RC(rc)); D_GOTO(out, rc); diff --git a/src/rebuild/rebuild_internal.h b/src/rebuild/rebuild_internal.h index 2b08c101675..c05af6c02e9 100644 --- a/src/rebuild/rebuild_internal.h +++ b/src/rebuild/rebuild_internal.h @@ -245,8 +245,13 @@ struct rebuild_task { */ uint32_t dst_reclaim_ver; + /* For failed tasks that may be retried after RB_OP_FAIL_RECLAIM, original map ver and opc + */ + uint32_t dst_retry_map_ver; + daos_rebuild_opc_t dst_retry_rebuild_op; + /* For dst_rebuild_op == RB_OP_FAIL_RECLAIM: If true, rebuild was stopped by admin. - * Then, on fail_reclaim finish, the pool rebuild state will be set to completed. + * Then, on fail_reclaim finish, the pool rebuild state will be set to idle (NOT_STARTED). */ bool dst_stop_admin; }; diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index af0ea5d937c..77b9bf44dd4 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -903,8 +903,13 @@ ds_rebuild_admin_stop(struct ds_pool *pool, uint32_t force) D_INFO(DF_RB ": NOT stopping rebuild during opc %u(%s)\n", DP_RB_RGT(rgt), rgt->rgt_opc, RB_OP_STR(rgt->rgt_opc)); } - rgt_put(rgt); + /* admin stop command does not terminate op:Fail_reclaim, but it is remembered to avoid + * retrying the original op:Rebuild. + */ + if (rgt->rgt_abort || (rgt->rgt_opc == RB_OP_FAIL_RECLAIM)) + rgt->rgt_stop_admin = 1; + rgt_put(rgt); return 0; } @@ -920,9 +925,9 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, unsigned int total; struct sched_req_attr attr = {0}; d_rank_t myrank; - unsigned int nsleeps = 0; - bool fi_admin_stop_fi = false; - bool fi_admin_stop = false; + unsigned int nsleeps = 0; + bool fi_admin_stop = false; + bool fi_admin_stop_frc = false; int rc; rc = crt_group_size(pool->sp_group, &total); @@ -947,11 +952,12 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, bool rebuild_abort = false; int i; - if (!fi_admin_stop_fi) - fi_admin_stop_fi = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP); - fi_admin_stop = (fi_admin_stop_fi && (op == RB_OP_REBUILD)); + if (!fi_admin_stop && (op == RB_OP_REBUILD)) + fi_admin_stop = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP); + if (!fi_admin_stop_frc && (op == RB_OP_FAIL_RECLAIM)) + fi_admin_stop_frc = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP_RECLAIM); - if (fi_admin_stop && (nsleeps > 0)) { + if ((fi_admin_stop || fi_admin_stop_frc) && (nsleeps > 0)) { uuid_copy(test_stop_start_puuid, rgt->rgt_pool_uuid); /* What the real RPC handler will call. We will exit below due to rgt_abort @@ -1054,7 +1060,7 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, sleep: update_and_warn_for_slow_engines(rgt); sched_req_sleep(rgt->rgt_ult, RBLD_CHECK_INTV); - if (fi_admin_stop) + if (fi_admin_stop || fi_admin_stop_frc) nsleeps++; } @@ -1674,12 +1680,12 @@ rebuild_leader_start(struct ds_pool *pool, struct rebuild_task *task, static void retry_rebuild_task(struct rebuild_task *task, struct rebuild_global_pool_tracker *rgt, - daos_rebuild_opc_t *opc) + bool do_map_revert, daos_rebuild_opc_t *opc) { int error = rgt->rgt_status.rs_errno; - int rc; + bool do_log = do_map_revert; - /* Only be called if rebuild task failed */ + /* Only called if task failed. Whether to log is also based on do_map_revert argument */ /* Do not retry if the administrator manually stopped the rebuild */ if (rgt->rgt_stop_admin) { @@ -1692,8 +1698,10 @@ retry_rebuild_task(struct rebuild_task *task, struct rebuild_global_pool_tracker */ if (daos_crt_network_error(error) || error == -DER_TIMEDOUT || error == -DER_GRPVER || error == -DER_STALE || error == -DER_VOS_PARTIAL_UPDATE) { - DL_INFO(error, DF_UUID" opc %u/%u retry", DP_UUID(task->dst_pool_uuid), - task->dst_rebuild_op, task->dst_map_ver); + if (do_log) + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u retry", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, + task->dst_map_ver); *opc = task->dst_rebuild_op; return; } @@ -1701,37 +1709,70 @@ retry_rebuild_task(struct rebuild_task *task, struct rebuild_global_pool_tracker /* Reclaim and exclude task needs to retry indefinitely */ if (task->dst_rebuild_op == RB_OP_RECLAIM || task->dst_rebuild_op == RB_OP_FAIL_RECLAIM) { - DL_INFO(error, DF_UUID" opc %u/%u retry", DP_UUID(task->dst_pool_uuid), - task->dst_rebuild_op, task->dst_map_ver); + if (do_log) + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u retry", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, + task->dst_map_ver); *opc = task->dst_rebuild_op; return; } /* Do not need retry for upgrade */ if (task->dst_rebuild_op == RB_OP_UPGRADE) { - DL_INFO(error, DF_UUID" opc %u/%u, no need to retry", DP_UUID(task->dst_pool_uuid), - task->dst_rebuild_op, task->dst_map_ver); + if (do_log) + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, + DF_UUID " opc %u/%u, no need to retry", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, + task->dst_map_ver); *opc = RB_OP_NONE; return; } - DL_INFO(error, DF_UUID" opc %u/%u, revert pool map", DP_UUID(task->dst_pool_uuid), - task->dst_rebuild_op, task->dst_map_ver); - rc = ds_pool_tgt_revert_rebuild(task->dst_pool_uuid, &task->dst_tgts); - if (rc < 0) - D_ERROR(DF_UUID" revert pool map status: %d\n", - DP_UUID(task->dst_pool_uuid), rc); + if (do_map_revert) { + int rc; + + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u, revert pool map", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); + + rc = ds_pool_tgt_revert_rebuild(task->dst_pool_uuid, &task->dst_tgts); + if (rc < 0) + D_ERROR(DF_UUID " revert pool map status: %d\n", + DP_UUID(task->dst_pool_uuid), rc); + } /* Though there might be DOWN targets in the list needs to retry anyway, * NB: those drain/extend/reintegration targets have been revert to UPIN * or DOWNOUT status, so during retry, rebuild job will be canceled anyway. * Only those exclude targets will keep retry here. */ - DL_INFO(error, DF_UUID" opc %u/%u, retry.", DP_UUID(task->dst_pool_uuid), - task->dst_rebuild_op, task->dst_map_ver); + if (do_log) + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u, retry.", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); *opc = RB_OP_REBUILD; } +static void +check_to_retry_orig_rebuild(struct rebuild_task *task, struct rebuild_global_pool_tracker *rgt) +{ + /* Only called for Fail_reclaim (whether it fails or succeeds) to know if the original + * op:Rebuild should be retried later when Fail_reclaim is (successfully) done. + */ + if (task->dst_rebuild_op != RB_OP_FAIL_RECLAIM) + return; + + /* If this Fail_reclaim was stopped, and the original Rebuild was not stopped, + * we no longer want the original Rebuild to be retried. + */ + if (rgt->rgt_stop_admin && (task->dst_retry_rebuild_op != RB_OP_NONE)) { + D_INFO( + DF_RB + ": rebuild stop command during Fail_reclaim - do NOT retry original %u(%s)\n", + DP_RB_RGT(rgt), task->dst_retry_rebuild_op, + RB_OP_STR(task->dst_retry_rebuild_op)); + task->dst_retry_rebuild_op = RB_OP_NONE; + } +} + static int rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, struct rebuild_global_pool_tracker *rgt, int ret) @@ -1752,8 +1793,9 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, rc = ds_rebuild_schedule(pool, task->dst_map_ver, task->dst_reclaim_eph, task->dst_new_layout_version, &task->dst_tgts, - task->dst_rebuild_op, task->dst_stop_admin, delay_sec); - DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_UUID ": schedule retry opc %u(%s)", + task->dst_rebuild_op, task->dst_retry_rebuild_op, + task->dst_retry_map_ver, task->dst_stop_admin, delay_sec); + DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_UUID ": reschedule not-started %u(%s)", DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, RB_OP_STR(task->dst_rebuild_op)); return rc; @@ -1770,8 +1812,11 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, DP_UUID(task->dst_pool_uuid), DP_RC(rc1)); } + /* see if we got stop during Fail_reclaim, configure whether to retry original rebuild */ + check_to_retry_orig_rebuild(task, rgt); + if (!is_rebuild_global_done(rgt) || rgt->rgt_status.rs_errno != 0) { - daos_rebuild_opc_t retry_opc = 0; + daos_rebuild_opc_t retry_opc = 0; /* If current job failed */ rgt->rgt_status.rs_state = DRS_IN_PROGRESS; @@ -1781,17 +1826,22 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, task->dst_rebuild_op == RB_OP_FAIL_RECLAIM) { rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_stable_epoch, task->dst_new_layout_version, &task->dst_tgts, - task->dst_rebuild_op, task->dst_stop_admin, + task->dst_rebuild_op, task->dst_retry_rebuild_op, + task->dst_retry_map_ver, task->dst_stop_admin, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, - DF_RB ": errno " DF_RC ", schedule retry", DP_RB_RGT(rgt), - DP_RC(rgt->rgt_status.rs_errno)); + DF_RB ": errno " DF_RC ", schedule retry %u(%s)", DP_RB_RGT(rgt), + DP_RC(rgt->rgt_status.rs_errno), task->dst_rebuild_op, + RB_OP_STR(task->dst_rebuild_op)); return rc; } - /* Schedule fail_reclaim to clean up current op. Retry original rebuild, even if - * this fails. However, do not retry the original rebuild if the admin has stopped - * it. + /* Determine if original rebuild needs a retry (don't revert pool map yet - see + * below) */ + retry_rebuild_task(task, rgt, false /* do_map_revert */, &retry_opc); + + /* Schedule fail_reclaim to clean up current op. After fail_reclaim, we may retry + * original. Scheduling any retry is deferred until fail_reclaim is actually done. */ if (rgt->rgt_init_scan) { /* NB: dst_reclaim_ver is the minimum rebuild target version, once rebuild @@ -1799,25 +1849,36 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, * (reclaim - 1 see obj_reclaim()), but keep the in-flight I/O data. */ rc = ds_rebuild_schedule( - pool, task->dst_reclaim_ver - 1, rgt->rgt_stable_epoch, + pool, task->dst_reclaim_ver - 1 /* map_ver */, rgt->rgt_stable_epoch, task->dst_new_layout_version, &task->dst_tgts, RB_OP_FAIL_RECLAIM, + retry_opc /* retry_rebuild_op */, task->dst_map_ver /* retry_map_ver */, rgt->rgt_stop_admin, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, - DF_RB ": errno " DF_RC ", schedule %u(%s)", DP_RB_RGT(rgt), - DP_RC(rgt->rgt_status.rs_errno), RB_OP_FAIL_RECLAIM, - RB_OP_STR(RB_OP_FAIL_RECLAIM)); + DF_RB ": errno " DF_RC + ", schedule %u(%s) and %s retry of original %u(%s)", + DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), + RB_OP_FAIL_RECLAIM, RB_OP_STR(RB_OP_FAIL_RECLAIM), + (retry_opc == RB_OP_NONE) ? "do NOT schedule" : "defer schedule", + task->dst_rebuild_op, RB_OP_STR(task->dst_rebuild_op)); + D_GOTO(complete, rc); } - /* Then check if it needs to retry */ - retry_rebuild_task(task, rgt, &retry_opc); + /* Determine if original rebuild needs a retry, and if so revert pool map. */ + retry_rebuild_task(task, rgt, true /* do_map_revert */, &retry_opc); + + /* Schedule a retry if needed (NB: possibly deferred until after a Fail_reclaim is + * done) */ if (retry_opc == RB_OP_NONE) D_GOTO(complete, rc); rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_stable_epoch, - task->dst_new_layout_version, &task->dst_tgts, retry_opc, - false /* stop_admin */, delay_sec); - DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule retry", - DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno)); + task->dst_new_layout_version, &task->dst_tgts, + retry_opc /* rebuild_op*/, 0 /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, delay_sec); + DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, + DF_RB ": errno " DF_RC ", schedule retry of original %u(%s)", + DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), retry_opc, + RB_OP_STR(retry_opc)); } else if (task->dst_rebuild_op == RB_OP_REBUILD || task->dst_rebuild_op == RB_OP_UPGRADE) { /* Schedule reclaim for successful op:Rebuild * (exclude/drain/reintegrate/extend/upgrade). */ @@ -1825,7 +1886,8 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_reclaim_epoch, task->dst_new_layout_version, &task->dst_tgts, - RB_OP_RECLAIM, false /* stop_admin */, delay_sec); + RB_OP_RECLAIM, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), RB_OP_RECLAIM, RB_OP_STR(RB_OP_RECLAIM)); @@ -1834,13 +1896,21 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, complete: /* Update the rebuild complete status for pool query */ if (task->dst_rebuild_op != RB_OP_FAIL_RECLAIM) { + /* update pool query rebuild status: original job is done. + * if op:Reclaim scheduled --> DRS_IN_PROGRESS, otherwise DRS_COMPLETED + */ rc1 = rebuild_status_completed_update(task->dst_pool_uuid, &rgt->rgt_status); DL_CDEBUG(rc1, DLOG_ERR, DLOG_INFO, rc1, DF_RB ": updated, state %d errno " DF_RC, DP_RB_RGT(rgt), rgt->rgt_status.rs_state, DP_RC(rgt->rgt_status.rs_errno)); - } else if (task->dst_rebuild_op == RB_OP_FAIL_RECLAIM && task->dst_stop_admin) { - /* Fail_reclaim normally does not update status - except in the manual admin stop - * case */ + } else if (task->dst_rebuild_op == RB_OP_FAIL_RECLAIM && + (task->dst_stop_admin || rgt->rgt_stop_admin)) { + /* update pool query rebuild status: either the original job was stopped, + * or a stop command was received during Fail_reclaim */ + if (rgt->rgt_stop_admin) + D_INFO(DF_RB ": stop received during Fail_reclaim, change errno=%d to %d\n", + DP_RB_RGT(rgt), rgt->rgt_status.rs_errno, -DER_OP_CANCELED); + rgt->rgt_status.rs_errno = -DER_OP_CANCELED; rgt->rgt_status.rs_state = DRS_NOT_STARTED; rc1 = rebuild_status_completed_update_partial( @@ -1848,6 +1918,19 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, DL_CDEBUG(rc1, DLOG_ERR, DLOG_INFO, rc1, DF_RB ": updated, state %d errno " DF_RC, DP_RB_RGT(rgt), rgt->rgt_status.rs_state, DP_RC(rgt->rgt_status.rs_errno)); + } else if ((task->dst_rebuild_op == RB_OP_FAIL_RECLAIM) && + (task->dst_retry_rebuild_op != RB_OP_NONE)) { + /* Fail_reclaim done (and a stop command wasn't received during) - retry original + * rebuild */ + rc1 = ds_rebuild_schedule(pool, task->dst_retry_map_ver, rgt->rgt_reclaim_epoch, + task->dst_new_layout_version, &task->dst_tgts, + task->dst_retry_rebuild_op, + RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */, + false /* stop_admin */, delay_sec); + DL_CDEBUG(rc1, DLOG_ERR, DLOG_INFO, rc, + DF_RB ": errno " DF_RC ", schedule retry %u(%s)", DP_RB_RGT(rgt), + DP_RC(rgt->rgt_status.rs_errno), task->dst_retry_rebuild_op, + RB_OP_STR(task->dst_retry_rebuild_op)); } if ((rc1 != 0) && (rc == 0)) @@ -2284,7 +2367,8 @@ rebuild_print_list_update(const uuid_t uuid, const uint32_t map_ver, int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t reclaim_eph, uint32_t layout_version, struct pool_target_id_list *tgts, - daos_rebuild_opc_t rebuild_op, bool stop_admin, uint64_t delay_sec) + daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op, + uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec) { struct rebuild_task *new_task; struct rebuild_task *task; @@ -2293,6 +2377,7 @@ ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t reclaim uint64_t cur_ts = 0; D_ASSERT(dss_get_module_info()->dmi_xs_id == 0); + D_ASSERT(rebuild_op != RB_OP_NONE); if (pool->sp_stopping) { D_DEBUG(DB_REBUILD, DF_UUID " is stopping, do not need schedule here\n", DP_UUID(pool->sp_uuid)); @@ -2330,7 +2415,15 @@ ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t reclaim new_task->dst_rebuild_op = rebuild_op; new_task->dst_reclaim_eph = reclaim_eph; new_task->dst_new_layout_version = layout_version; - new_task->dst_stop_admin = (rebuild_op == RB_OP_FAIL_RECLAIM) ? stop_admin : false; + if (rebuild_op == RB_OP_FAIL_RECLAIM) { + new_task->dst_retry_map_ver = retry_map_ver; + new_task->dst_retry_rebuild_op = retry_rebuild_op; + new_task->dst_stop_admin = stop_admin; + } else { + new_task->dst_retry_map_ver = 0; + new_task->dst_retry_rebuild_op = RB_OP_NONE; + new_task->dst_stop_admin = false; + } uuid_copy(new_task->dst_pool_uuid, pool->sp_uuid); D_INIT_LIST_HEAD(&new_task->dst_list); @@ -2417,14 +2510,16 @@ regenerate_task_internal(struct ds_pool *pool, struct pool_target *tgts, id_list.pti_number = 1; if (tgt->ta_comp.co_status & (PO_COMP_ST_DOWN | PO_COMP_ST_DRAIN)) { - rc = ds_rebuild_schedule(pool, tgt->ta_comp.co_fseq, - current_eph == 0 ? eph : current_eph, 0, &id_list, - RB_OP_REBUILD, false /* stop_admin */, delay); + rc = ds_rebuild_schedule( + pool, tgt->ta_comp.co_fseq, current_eph == 0 ? eph : current_eph, 0, + &id_list, RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, delay); } else { D_ASSERT(tgt->ta_comp.co_status == PO_COMP_ST_UP); - rc = ds_rebuild_schedule(pool, tgt->ta_comp.co_in_ver, - current_eph == 0 ? eph : current_eph, 0, &id_list, - RB_OP_REBUILD, false /* stop_admin */, delay); + rc = ds_rebuild_schedule( + pool, tgt->ta_comp.co_in_ver, current_eph == 0 ? eph : current_eph, 0, + &id_list, RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, delay); } if (rc) { D_ERROR(DF_UUID" schedule ver %d failed: "DF_RC"\n", diff --git a/src/tests/suite/daos_rebuild_common.c b/src/tests/suite/daos_rebuild_common.c index f65f0ad4f41..5ecd1e0439d 100644 --- a/src/tests/suite/daos_rebuild_common.c +++ b/src/tests/suite/daos_rebuild_common.c @@ -1207,6 +1207,31 @@ fi_rebuild_stop(void *data) return 0; } +/* inject fault on leader engine to receive stop command during op:Fail_reclaim */ +int +fi_rebuild_stop_reclaim(void *data) +{ + test_arg_t *arg = data; + d_rank_t leader; + int rc; + + if (arg->myrank != 0) + return 0; + + rc = test_get_leader(arg, &leader); + if (rc) { + print_message("get pool " DF_UUIDF " leader failed: %d\n", + DP_UUID(arg->pool.pool_uuid), rc); + assert_rc_equal(rc, 0); + } + + printf("%s(): pool " DF_UUIDF ": send ADMIN_STOP_RECLAIM to leader rank %d\n", __FUNCTION__, + DP_UUID(arg->pool.pool_uuid), leader); + test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_STOP_RECLAIM | DAOS_FAIL_ONCE); + + return 0; +} + /* resume after a stopped rebuild, and optionally inject a fault to start a new rebuild */ int fi_rebuild_resume_wait(void *data) diff --git a/src/tests/suite/daos_rebuild_simple.c b/src/tests/suite/daos_rebuild_simple.c index 85c05dc41ad..6b95dc09047 100644 --- a/src/tests/suite/daos_rebuild_simple.c +++ b/src/tests/suite/daos_rebuild_simple.c @@ -1475,9 +1475,11 @@ rebuild_wait_reset_fail_cb(void *data) { test_arg_t *arg = data; - print_message("wait 300 seconds for rebuild/reclaim/retry...."); + print_message("wait 60 seconds for rebuild/reclaim%s\n", + arg->interactive_rebuild ? "" : "/retry"); sleep(60); + print_message("clearing fault injection on all engines\n"); daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0, NULL); daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 0, 0, NULL); daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_NUM, 0, 0, NULL); @@ -1496,11 +1498,6 @@ rebuild_many_objects_with_failure(void **state) if (!test_runable(arg, 6)) return; - if (arg->interactive_rebuild) { - print_message("SKIP due to interactive_rebuild enabled, but not tested here\n"); - skip(); - } - D_ALLOC_ARRAY(oids, 8000); for (i = 0; i < 8000; i++) { char buffer[256]; @@ -1517,15 +1514,48 @@ rebuild_many_objects_with_failure(void **state) ioreq_fini(&req); } + /* Inject faults on engines. Special handling for interactive_rebuild case */ if (arg->myrank == 0) { - daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, - DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); - daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 50, - 0, NULL); + if (!arg->interactive_rebuild) { + /* All engines DAOS_REBUILD_OBJ_FAIL */ + print_message("inject fault DAOS_REBUILD_OBJ_FAIL on all engines\n"); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, + DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 50, 0, NULL); + } else { + d_rank_t leader; + + /* interactive_rebiuld: DAOS_REBUILD_OBJ fail on all engines except PS + * leader; DAOS_REBUILD_ADMIN_STOP_RECLAIM on the PS leader engine + */ + + rc = test_get_leader(arg, &leader); + if (rc != 0) { + print_message("get pool " DF_UUIDF " leader failed: %d\n", + DP_UUID(arg->pool.pool_uuid), rc); + assert_rc_equal(rc, 0); + } + + for (i = 0; i < arg->srv_nnodes; i++) { + if (i == leader) + fi_rebuild_stop_reclaim(arg); + else { + print_message("inject DAOS_REBUILD_OBJ_FAIL on non-leader " + "engine %u\n", + i); + daos_debug_set_params( + arg->group, i, DMG_KEY_FAIL_LOC, + DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); + daos_debug_set_params(arg->group, i, DMG_KEY_FAIL_VALUE, 50, + 0, NULL); + } + } + } } arg->rebuild_cb = rebuild_wait_reset_fail_cb; - + if (arg->interactive_rebuild) + arg->rebuild_post_cb = fi_rebuild_resume_wait; rebuild_single_pool_target(arg, 3, -1, false); for (i = 0; i < 8000; i++) { diff --git a/src/tests/suite/daos_test.h b/src/tests/suite/daos_test.h index f7ddeb7ff84..97a56bdec7d 100644 --- a/src/tests/suite/daos_test.h +++ b/src/tests/suite/daos_test.h @@ -470,6 +470,8 @@ int rebuild_sub_setup_common(void **state, daos_size_t pool_size, int node_nr, u int fi_rebuild_stop(void *data); +int +fi_rebuild_stop_reclaim(void *data); int fi_rebuild_resume_wait(void *data); From 10a2d3fdde2fef2e8f2fba87c30f54c767873759 Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Wed, 10 Sep 2025 21:47:57 -0400 Subject: [PATCH 2/4] Address review comments and candidate test fix daos_test -B --subtests="15" EXTEND16 failure Features: rebuild Allow-unstable-test: true Signed-off-by: Kenneth Cain --- src/rebuild/srv.c | 63 +++++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index c5373de83a2..415b7df791f 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -898,7 +898,6 @@ ds_rebuild_admin_stop(struct ds_pool *pool, uint32_t force) D_INFO(DF_RB ": stopping rebuild force=%u opc %u(%s)\n", DP_RB_RGT(rgt), force, rgt->rgt_opc, RB_OP_STR(rgt->rgt_opc)); rgt->rgt_abort = 1; - rgt->rgt_stop_admin = 1; rgt->rgt_status.rs_errno = -DER_OP_CANCELED; } else { D_INFO(DF_RB ": NOT stopping rebuild during opc %u(%s)\n", DP_RB_RGT(rgt), @@ -1681,10 +1680,10 @@ rebuild_leader_start(struct ds_pool *pool, struct rebuild_task *task, static void retry_rebuild_task(struct rebuild_task *task, struct rebuild_global_pool_tracker *rgt, - bool do_map_revert, daos_rebuild_opc_t *opc) + daos_rebuild_opc_t *opc) { + int rc; int error = rgt->rgt_status.rs_errno; - bool do_log = do_map_revert; /* Only called if task failed. Whether to log is also based on do_map_revert argument */ @@ -1699,10 +1698,8 @@ retry_rebuild_task(struct rebuild_task *task, struct rebuild_global_pool_tracker */ if (daos_crt_network_error(error) || error == -DER_TIMEDOUT || error == -DER_GRPVER || error == -DER_STALE || error == -DER_VOS_PARTIAL_UPDATE) { - if (do_log) - DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u retry", - DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, - task->dst_map_ver); + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u retry", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); *opc = task->dst_rebuild_op; return; } @@ -1710,45 +1707,34 @@ retry_rebuild_task(struct rebuild_task *task, struct rebuild_global_pool_tracker /* Reclaim and exclude task needs to retry indefinitely */ if (task->dst_rebuild_op == RB_OP_RECLAIM || task->dst_rebuild_op == RB_OP_FAIL_RECLAIM) { - if (do_log) - DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u retry", - DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, - task->dst_map_ver); + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u retry", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); *opc = task->dst_rebuild_op; return; } /* Do not need retry for upgrade */ if (task->dst_rebuild_op == RB_OP_UPGRADE) { - if (do_log) - DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, - DF_UUID " opc %u/%u, no need to retry", - DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, - task->dst_map_ver); + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u, no need to retry", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); *opc = RB_OP_NONE; return; } - if (do_map_revert) { - int rc; + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u, revert pool map", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); - DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u, revert pool map", - DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); - - rc = ds_pool_tgt_revert_rebuild(task->dst_pool_uuid, &task->dst_tgts); - if (rc < 0) - D_ERROR(DF_UUID " revert pool map status: %d\n", - DP_UUID(task->dst_pool_uuid), rc); - } + rc = ds_pool_tgt_revert_rebuild(task->dst_pool_uuid, &task->dst_tgts); + if (rc < 0) + D_ERROR(DF_UUID " revert pool map status: %d\n", DP_UUID(task->dst_pool_uuid), rc); /* Though there might be DOWN targets in the list needs to retry anyway, * NB: those drain/extend/reintegration targets have been revert to UPIN * or DOWNOUT status, so during retry, rebuild job will be canceled anyway. * Only those exclude targets will keep retry here. */ - if (do_log) - DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u, retry.", - DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); + DL_CDEBUG(error, DLOG_ERR, DLOG_INFO, error, DF_UUID " opc %u/%u, retry.", + DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, task->dst_map_ver); *opc = RB_OP_REBUILD; } @@ -1837,10 +1823,6 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, return rc; } - /* Determine if original rebuild needs a retry (don't revert pool map yet - see - * below) */ - retry_rebuild_task(task, rgt, false /* do_map_revert */, &retry_opc); - /* Schedule fail_reclaim to clean up current op. After fail_reclaim, we may retry * original. Scheduling any retry is deferred until fail_reclaim is actually done. */ @@ -1855,17 +1837,18 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, retry_opc /* retry_rebuild_op */, task->dst_map_ver /* retry_map_ver */, rgt->rgt_stop_admin, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, - DF_RB ": errno " DF_RC - ", schedule %u(%s) and %s retry of original %u(%s)", - DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), - RB_OP_FAIL_RECLAIM, RB_OP_STR(RB_OP_FAIL_RECLAIM), - (retry_opc == RB_OP_NONE) ? "do NOT schedule" : "defer schedule", - task->dst_rebuild_op, RB_OP_STR(task->dst_rebuild_op)); + DF_RB ": errno " DF_RC ", schedule %u(%s)", DP_RB_RGT(rgt), + DP_RC(rgt->rgt_status.rs_errno), RB_OP_FAIL_RECLAIM, + RB_OP_STR(RB_OP_FAIL_RECLAIM)); + + /* revert pool map and defer scheduling a retry until Fail_reclaim is done + */ + retry_rebuild_task(task, rgt, &retry_opc); D_GOTO(complete, rc); } /* Determine if original rebuild needs a retry, and if so revert pool map. */ - retry_rebuild_task(task, rgt, true /* do_map_revert */, &retry_opc); + retry_rebuild_task(task, rgt, &retry_opc); /* Schedule a retry if needed (NB: possibly deferred until after a Fail_reclaim is * done) */ From 7b02632657d7164094a3d69c38291d580c3e8fd9 Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Fri, 19 Sep 2025 12:32:23 -0400 Subject: [PATCH 3/4] DAOS-17357 rebuild: use dmg for stop/start, force-stop fix Remove use of fault injection for interactive rebuild testing in daos_test -v (rebuild simple tests). Replace with new dmg command helper functions to invoke the control plane commands dmg pool rebuild stop and dmg pool rebuild start. Some common test functions are added and modified to support this. Also, some rebuild engine logic is updated to remove the associated fault injection handling. Restore the engine rebuild_ults() logic so that it does not loop forever (a change made in initial implementation for fault injection). Also, update the engine rebuild stop logic so it does not allow "dmg pool rebuild stop --force" to stop a rebuild in op:Fail_reclaim, unless it has been failing repeatedly. For this, new counts are added in struct rebuild_task and struct rebulid_global_pool_tracker to track, for a high level rebuild operation, how many times op:Rebuild, Reclaim, Fail_reclaim, etc. have been run (and how many times op:Rebuild and op:Fail_reclaim have failed). daos_test -v rebuild_many_objects_with_failure() is updated to specifically use the --force option to exercise arrival of command during Fail_reclaim, and exercise that the rebuild will not be stopped during Fail_reclaim in this instance. i.e., it will work the same as without the --force option, preventing retry of original op:Rebuild after the one/only Fail_reclaim is done. Test-tag: pr test_daos_rebuild_simple_interactive Allow-unstable-test: true Signed-off-by: Kenneth Cain --- src/common/tests_dmg_helpers.c | 74 +++++++++ src/include/daos/common.h | 3 - src/include/daos/tests_lib.h | 19 +++ src/include/daos_srv/rebuild.h | 2 +- src/pool/srv_pool.c | 5 +- src/rebuild/rebuild_internal.h | 13 ++ src/rebuild/srv.c | 161 +++++++++++-------- src/tests/suite/daos_rebuild_common.c | 140 ++++++++++------- src/tests/suite/daos_rebuild_simple.c | 214 +++++++++++++++----------- src/tests/suite/daos_test.h | 44 ++++-- src/tests/suite/daos_test_common.c | 108 ++++++++++++- 11 files changed, 545 insertions(+), 238 deletions(-) diff --git a/src/common/tests_dmg_helpers.c b/src/common/tests_dmg_helpers.c index dcf1b274116..08825366133 100644 --- a/src/common/tests_dmg_helpers.c +++ b/src/common/tests_dmg_helpers.c @@ -1206,6 +1206,80 @@ dmg_pool_list(const char *dmg_config_file, const char *group, return rc; } +int +dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force) +{ + char uuid_str[DAOS_UUID_STR_SIZE]; + int argcount = 0; + char **args = NULL; + struct json_object *dmg_out = NULL; + int rc = 0; + + uuid_unparse_lower(uuid, uuid_str); + args = cmd_push_arg(args, &argcount, "%s ", uuid_str); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + + if (grp != NULL) { + args = cmd_push_arg(args, &argcount, "--sys=%s ", grp); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + if (force) { + args = cmd_push_arg(args, &argcount, "--force"); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + rc = daos_dmg_json_pipe("pool rebuild stop", dmg_config_file, args, argcount, &dmg_out); + if (rc != 0) { + D_ERROR("dmg pool rebuild stop failed\n"); + goto out_json; + } + +out_json: + if (dmg_out != NULL) + json_object_put(dmg_out); + cmd_free_args(args, argcount); +out: + return rc; +} + +int +dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp) +{ + char uuid_str[DAOS_UUID_STR_SIZE]; + int argcount = 0; + char **args = NULL; + struct json_object *dmg_out = NULL; + int rc = 0; + + uuid_unparse_lower(uuid, uuid_str); + args = cmd_push_arg(args, &argcount, "%s ", uuid_str); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + + if (grp != NULL) { + args = cmd_push_arg(args, &argcount, "--sys=%s ", grp); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + rc = daos_dmg_json_pipe("pool rebuild start", dmg_config_file, args, argcount, &dmg_out); + if (rc != 0) { + D_ERROR("dmg pool rebuild start failed\n"); + goto out_json; + } + +out_json: + if (dmg_out != NULL) + json_object_put(dmg_out); + cmd_free_args(args, argcount); +out: + return rc; +} + static int parse_device_info(struct json_object *smd_dev, device_list *devices, char *host, int dev_length, int *disks) diff --git a/src/include/daos/common.h b/src/include/daos/common.h index 7999879193a..6c1d24fd1ac 100644 --- a/src/include/daos/common.h +++ b/src/include/daos/common.h @@ -789,9 +789,6 @@ enum { #define DAOS_RDB_SKIP_APPENDENTRIES_FAIL (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1a) #define DAOS_FORCE_REFRESH_POOL_MAP (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1b) -#define DAOS_REBUILD_ADMIN_STOP (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1c) -#define DAOS_REBUILD_ADMIN_START (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1d) - #define DAOS_FORCE_CAPA_FETCH (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1e) #define DAOS_FORCE_PROP_VERIFY (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1f) diff --git a/src/include/daos/tests_lib.h b/src/include/daos/tests_lib.h index 8c9ed5bf1f9..72438598297 100644 --- a/src/include/daos/tests_lib.h +++ b/src/include/daos/tests_lib.h @@ -484,6 +484,25 @@ dmg_pool_set_prop(const char *dmg_config_file, int dmg_pool_get_prop(const char *dmg_config_file, const char *label, const uuid_t uuid, const char *name, char **value); +/** + * Interactively stop a pool's currently-running rebuild. + * \param dmg_config_file [IN] DMG config file. + * \param uuid [IN] UUID of the pool. + * \param grp [IN] Process set name of the DAOS servers managing the pool. + * \param force [IN] forcibly stop a rebuild that is failing. + */ +int +dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force); + +/** + * Interactively start/resume a pool's rebuilding. + * \param dmg_config_file [IN] DMG config file. + * \param uuid [IN] UUID of the pool. + * \param grp [IN] Process set name of the DAOS servers managing the pool. + */ +int + dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp); + /** * List all disks in the specified DAOS system. * diff --git a/src/include/daos_srv/rebuild.h b/src/include/daos_srv/rebuild.h index d5714314bbd..553f1a6ff45 100644 --- a/src/include/daos_srv/rebuild.h +++ b/src/include/daos_srv/rebuild.h @@ -87,7 +87,7 @@ int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t stable_eph, uint32_t layout_version, struct pool_target_id_list *tgts, daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op, - uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec); + uint32_t retry_map_ver, bool stop_admin, void *cur_taskp, uint64_t delay_sec); void ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank); int ds_rebuild_query(uuid_t pool_uuid, struct daos_rebuild_status *status); diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 276b311c4a2..5441d52cf00 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -6410,7 +6410,7 @@ pool_check_upgrade_object_layout(struct rdb_tx *tx, struct pool_svc *svc, rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, upgrade_eph, DS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE, RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */, - false /* stop_admin */, 0); + false /* stop_admin */, NULL /* cur_taskp */, 0); if (rc == 0) *scheduled_layout_upgrade = true; } @@ -7758,7 +7758,8 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank, if (tgt_map_ver != 0) { rc = ds_rebuild_schedule(svc->ps_pool, tgt_map_ver, rebuild_eph, 0, &target_list, RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay); + 0 /* retry_map_ver */, false /* stop_admin */, + NULL /* cur_taskp */, delay); if (rc != 0) { D_ERROR("rebuild fails rc: "DF_RC"\n", DP_RC(rc)); D_GOTO(out, rc); diff --git a/src/rebuild/rebuild_internal.h b/src/rebuild/rebuild_internal.h index c05af6c02e9..14564dbf746 100644 --- a/src/rebuild/rebuild_internal.h +++ b/src/rebuild/rebuild_internal.h @@ -168,6 +168,11 @@ struct rebuild_global_pool_tracker { uint32_t rgt_opc; unsigned int rgt_abort : 1, /* abort: kill rebuild */ rgt_init_scan : 1, rgt_stop_admin : 1; /* stop: admin has asked to kill rebuild */ + + uint32_t rgt_num_op_rb; /* count of op:Rebuild attempts */ + uint32_t rgt_num_op_freclaim; /* count of op:Fail_reclaim attempts */ + uint32_t rgt_num_op_rb_fail; /* count of op:Rebuild failures */ + uint32_t rgt_num_op_freclaim_fail; /* count of failed op:Fail_reclaim (not good) */ }; /* Structure on raft replica nodes to serve completed rebuild status querying */ @@ -254,6 +259,14 @@ struct rebuild_task { * Then, on fail_reclaim finish, the pool rebuild state will be set to idle (NOT_STARTED). */ bool dst_stop_admin; + + /* Track how many tries for certain daos_rebuild_opc_t */ + uint32_t dst_num_op_rb; /* count of tries to run rebuild */ + uint32_t dst_num_op_reclaim; + uint32_t dst_num_op_freclaim; + uint32_t dst_num_op_upgrade; + uint32_t dst_num_op_rb_fail; /* count of rebuild failures */ + uint32_t dst_num_op_freclaim_fail; /* count of Fail_recliam failures */ }; /* Per pool structure in TLS to check pool rebuild status diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index 415b7df791f..7a51396ddf8 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -874,8 +874,17 @@ enum { RB_BCAST_QUERY, }; -/* for interactive mode rebuild fault-injection testing */ -static uuid_t test_stop_start_puuid; +static bool +rebuild_is_stoppable(struct rebuild_global_pool_tracker *rgt, bool force) +{ + if ((rgt->rgt_opc == RB_OP_REBUILD) || (rgt->rgt_opc == RB_OP_UPGRADE)) + return true; + + if ((rgt->rgt_opc == RB_OP_FAIL_RECLAIM) && force && (rgt->rgt_num_op_freclaim_fail > 0)) + return true; + + return false; +} /* administrator (via dmg command) asks to stop the currently-running rebuild for pool */ int @@ -893,8 +902,7 @@ ds_rebuild_admin_stop(struct ds_pool *pool, uint32_t force) } /* admin stop command does not terminate reclaim/fail_reclaim jobs (unless forced) */ - if (((rgt->rgt_opc == RB_OP_REBUILD) || (rgt->rgt_opc == RB_OP_UPGRADE)) || - ((rgt->rgt_opc == RB_OP_FAIL_RECLAIM) && force)) { + if (rebuild_is_stoppable(rgt, force)) { D_INFO(DF_RB ": stopping rebuild force=%u opc %u(%s)\n", DP_RB_RGT(rgt), force, rgt->rgt_opc, RB_OP_STR(rgt->rgt_opc)); rgt->rgt_abort = 1; @@ -925,9 +933,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, unsigned int total; struct sched_req_attr attr = {0}; d_rank_t myrank; - unsigned int nsleeps = 0; - bool fi_admin_stop = false; - bool fi_admin_stop_frc = false; int rc; rc = crt_group_size(pool->sp_group, &total); @@ -952,19 +957,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, bool rebuild_abort = false; int i; - if (!fi_admin_stop && (op == RB_OP_REBUILD)) - fi_admin_stop = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP); - if (!fi_admin_stop_frc && (op == RB_OP_FAIL_RECLAIM)) - fi_admin_stop_frc = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP_RECLAIM); - - if ((fi_admin_stop || fi_admin_stop_frc) && (nsleeps > 0)) { - uuid_copy(test_stop_start_puuid, rgt->rgt_pool_uuid); - - /* What the real RPC handler will call. We will exit below due to rgt_abort - */ - ds_pool_rebuild_stop(test_stop_start_puuid, 0 /* force */, NULL /* hint */); - } - ABT_rwlock_rdlock(pool->sp_lock); rc = map_ranks_init(pool->sp_map, PO_COMP_ST_UP | PO_COMP_ST_DOWN | @@ -1060,8 +1052,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, sleep: update_and_warn_for_slow_engines(rgt); sched_req_sleep(rgt->rgt_ult, RBLD_CHECK_INTV); - if (fi_admin_stop || fi_admin_stop_frc) - nsleeps++; } sched_req_put(rgt->rgt_ult); @@ -1665,6 +1655,10 @@ rebuild_leader_start(struct ds_pool *pool, struct rebuild_task *task, return rc; D_ASSERT(*p_rgt != NULL); + (*p_rgt)->rgt_num_op_rb = task->dst_num_op_rb; + (*p_rgt)->rgt_num_op_freclaim = task->dst_num_op_freclaim; + (*p_rgt)->rgt_num_op_rb_fail = task->dst_num_op_rb_fail; + (*p_rgt)->rgt_num_op_freclaim_fail = task->dst_num_op_freclaim_fail; D_INFO(DF_RB "\n", DP_RB_RGT(*p_rgt)); /* broadcast scan RPC to all targets */ @@ -1778,10 +1772,10 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, return 0; } - rc = ds_rebuild_schedule(pool, task->dst_map_ver, task->dst_reclaim_eph, - task->dst_new_layout_version, &task->dst_tgts, - task->dst_rebuild_op, task->dst_retry_rebuild_op, - task->dst_retry_map_ver, task->dst_stop_admin, delay_sec); + rc = ds_rebuild_schedule( + pool, task->dst_map_ver, task->dst_reclaim_eph, task->dst_new_layout_version, + &task->dst_tgts, task->dst_rebuild_op, task->dst_retry_rebuild_op, + task->dst_retry_map_ver, task->dst_stop_admin, NULL /* cur_taskp */, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_UUID ": reschedule not-started %u(%s)", DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, RB_OP_STR(task->dst_rebuild_op)); @@ -1815,7 +1809,7 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, task->dst_new_layout_version, &task->dst_tgts, task->dst_rebuild_op, task->dst_retry_rebuild_op, task->dst_retry_map_ver, task->dst_stop_admin, - delay_sec); + task, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule retry %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), task->dst_rebuild_op, @@ -1835,7 +1829,7 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, pool, task->dst_reclaim_ver - 1 /* map_ver */, rgt->rgt_stable_epoch, task->dst_new_layout_version, &task->dst_tgts, RB_OP_FAIL_RECLAIM, retry_opc /* retry_rebuild_op */, task->dst_map_ver /* retry_map_ver */, - rgt->rgt_stop_admin, delay_sec); + rgt->rgt_stop_admin, task, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), RB_OP_FAIL_RECLAIM, @@ -1855,10 +1849,10 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, if (retry_opc == RB_OP_NONE) D_GOTO(complete, rc); - rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_stable_epoch, - task->dst_new_layout_version, &task->dst_tgts, - retry_opc /* rebuild_op*/, 0 /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay_sec); + rc = ds_rebuild_schedule( + pool, task->dst_map_ver, rgt->rgt_stable_epoch, task->dst_new_layout_version, + &task->dst_tgts, retry_opc /* rebuild_op*/, 0 /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, task, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule retry of original %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), retry_opc, @@ -1868,10 +1862,10 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, * (exclude/drain/reintegrate/extend/upgrade). */ rgt->rgt_status.rs_state = DRS_IN_PROGRESS; - rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_reclaim_epoch, - task->dst_new_layout_version, &task->dst_tgts, - RB_OP_RECLAIM, RB_OP_NONE /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay_sec); + rc = ds_rebuild_schedule( + pool, task->dst_map_ver, rgt->rgt_reclaim_epoch, task->dst_new_layout_version, + &task->dst_tgts, RB_OP_RECLAIM, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, task, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), RB_OP_RECLAIM, RB_OP_STR(RB_OP_RECLAIM)); @@ -1910,7 +1904,7 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, task->dst_new_layout_version, &task->dst_tgts, task->dst_retry_rebuild_op, RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */, - false /* stop_admin */, delay_sec); + false /* stop_admin */, task, delay_sec); DL_CDEBUG(rc1, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule retry %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), task->dst_retry_rebuild_op, @@ -2130,25 +2124,13 @@ rebuild_ults(void *arg) while (DAOS_FAIL_CHECK(DAOS_REBUILD_HANG)) ABT_thread_yield(); - /* Stay alive for fault-injection possibility (need a ULT for DAOS_REBUILD_ADMIN_START) */ - while (true) { - bool fi_admin_start = false; - + while (!d_list_empty(&rebuild_gst.rg_queue_list) || + !d_list_empty(&rebuild_gst.rg_running_list)) { if (rebuild_gst.rg_abort) { D_INFO("abort rebuilds\n"); break; } - fi_admin_start = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_START); - if (fi_admin_start) { - if (!uuid_is_null(test_stop_start_puuid)) { - D_INFO("fi_admin_start=%d, apply to pool " DF_UUID "\n", - fi_admin_start, DP_UUID(test_stop_start_puuid)); - ds_pool_rebuild_start(test_stop_start_puuid, NULL /* hint */); - uuid_clear(test_stop_start_puuid); - } - } - if (d_list_empty(&rebuild_gst.rg_queue_list) || rebuild_gst.rg_inflight >= REBUILD_MAX_INFLIGHT) { D_INFO("in-flight rebuild %u - going to sleep for 5 seconds\n", @@ -2352,8 +2334,9 @@ int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t reclaim_eph, uint32_t layout_version, struct pool_target_id_list *tgts, daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op, - uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec) + uint32_t retry_map_ver, bool stop_admin, void *cur_taskp, uint64_t delay_sec) { + struct rebuild_task *cur_task = cur_taskp; struct rebuild_task *new_task; struct rebuild_task *task; d_list_t *inserted_pos; @@ -2408,6 +2391,60 @@ ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t reclaim new_task->dst_retry_rebuild_op = RB_OP_NONE; new_task->dst_stop_admin = false; } + + /* update operation counts based on this (cur_task->dst_rebuild_op) -> new (rebuild_op), + * e.g.: op:Rebuild -> op:Reclaim or op:Fail_reclaim op:Reclaim -> op:Reclaim (unexpected, + * not tracked / Reclaim failure) op:Rebuild -> op:Rebuild (retry original rebuild / + * scheduled when original Rebuild fails) op:Fail_reclaim -> op:Rebuild (retry original + * rebuild / scheduled after Fail_reclaim done) op:Fail_reclaim -> op:Fail_reclaim + * (unexpected and tracked / Fail_reclaim failure) + */ + /* TODO: does upgrade ever happen with cur_task != NULL? */ + new_task->dst_num_op_rb = cur_task ? cur_task->dst_num_op_rb : 0; + new_task->dst_num_op_reclaim = cur_task ? cur_task->dst_num_op_reclaim : 0; + new_task->dst_num_op_freclaim = cur_task ? cur_task->dst_num_op_freclaim : 0; + new_task->dst_num_op_upgrade = cur_task ? cur_task->dst_num_op_upgrade : 0; + new_task->dst_num_op_rb_fail = cur_task ? cur_task->dst_num_op_rb_fail : 0; + new_task->dst_num_op_freclaim_fail = cur_task ? cur_task->dst_num_op_freclaim_fail : 0; + + switch (rebuild_op) { + case RB_OP_REBUILD: + new_task->dst_num_op_rb++; + break; + case RB_OP_RECLAIM: + D_ASSERT(cur_task != NULL); + new_task->dst_num_op_reclaim++; + break; + case RB_OP_FAIL_RECLAIM: + D_ASSERT(cur_task != NULL); + new_task->dst_num_op_freclaim++; + /* NB: update fail count in cur_task in case caller may be scheduling both + * Fail_reclaim and retry of op:Rebuild at the same time (i.e., while processing + * completion of the original/failed Rebuild). + */ + if (cur_task->dst_rebuild_op == RB_OP_REBUILD) + new_task->dst_num_op_rb_fail = ++cur_task->dst_num_op_rb_fail; + + /* Fail_reclaim failing repeatedly */ + if (cur_task->dst_rebuild_op == RB_OP_FAIL_RECLAIM) + new_task->dst_num_op_freclaim_fail++; + break; + case RB_OP_UPGRADE: + new_task->dst_num_op_upgrade++; + break; + default: + break; + } + + D_INFO(DF_UUID + ": %s%s%s (ver=%u), counts (rb/reclaim/freclaim/upgrade/rb_fail/freclaim_fail)" + "= %u/%u/%u/%u/%u/%u\n", + DP_UUID(pool->sp_uuid), cur_task ? RB_OP_STR(cur_task->dst_rebuild_op) : "", + cur_task ? "->" : "", RB_OP_STR(new_task->dst_rebuild_op), new_task->dst_map_ver, + new_task->dst_num_op_rb, new_task->dst_num_op_reclaim, new_task->dst_num_op_freclaim, + new_task->dst_num_op_upgrade, new_task->dst_num_op_rb_fail, + new_task->dst_num_op_freclaim_fail); + uuid_copy(new_task->dst_pool_uuid, pool->sp_uuid); D_INIT_LIST_HEAD(&new_task->dst_list); @@ -2494,16 +2531,18 @@ regenerate_task_internal(struct ds_pool *pool, struct pool_target *tgts, id_list.pti_number = 1; if (tgt->ta_comp.co_status & (PO_COMP_ST_DOWN | PO_COMP_ST_DRAIN)) { - rc = ds_rebuild_schedule( - pool, tgt->ta_comp.co_fseq, current_eph == 0 ? eph : current_eph, 0, - &id_list, RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay); + rc = ds_rebuild_schedule(pool, tgt->ta_comp.co_fseq, + current_eph == 0 ? eph : current_eph, 0, &id_list, + RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, + NULL /* cur_taskp */, delay); } else { D_ASSERT(tgt->ta_comp.co_status == PO_COMP_ST_UP); - rc = ds_rebuild_schedule( - pool, tgt->ta_comp.co_in_ver, current_eph == 0 ? eph : current_eph, 0, - &id_list, RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay); + rc = ds_rebuild_schedule(pool, tgt->ta_comp.co_in_ver, + current_eph == 0 ? eph : current_eph, 0, &id_list, + RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, + NULL /* cur_taskp */, delay); } if (rc) { D_ERROR(DF_UUID" schedule ver %d failed: "DF_RC"\n", @@ -3105,8 +3144,6 @@ init(void) { int rc; - uuid_clear(test_stop_start_puuid); - D_INIT_LIST_HEAD(&rebuild_gst.rg_tgt_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_global_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_completed_list); diff --git a/src/tests/suite/daos_rebuild_common.c b/src/tests/suite/daos_rebuild_common.c index 5ecd1e0439d..8f1bc7eb944 100644 --- a/src/tests/suite/daos_rebuild_common.c +++ b/src/tests/suite/daos_rebuild_common.c @@ -62,6 +62,8 @@ rebuild_exclude_tgt(test_arg_t **args, int arg_cnt, d_rank_t rank, for (i = 0; i < arg_cnt; i++) { rc = dmg_pool_exclude(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool exclude rank %u tgt_idx=%d " DF_UUID ", rc=%d\n", rank, + tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } } @@ -83,6 +85,9 @@ rebuild_reint_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_reintegrate(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool reintegrate rank %u tgt_idx=%d " DF_UUID + ", rc=%d\n", + rank, tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -100,6 +105,8 @@ rebuild_extend_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_extend(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, &rank, 1); + print_message("dmg pool extend rank %u " DF_UUID ", rc=%d\n", rank, + DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -117,6 +124,8 @@ rebuild_drain_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_drain(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool drain rank %u tgt_idx=%d " DF_UUID ", rc=%d\n", + rank, tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -162,16 +171,24 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, par_barrier(PAR_COMM_WORLD); for (i = 0; i < args_cnt; i++) - if (args[i]->rebuild_cb) + if (args[i]->rebuild_cb) { + print_message("call rebuild_cb for exclude rebuilding pool " DF_UUID + "\n", + DP_UUID(args[i]->pool.pool_uuid)); args[i]->rebuild_cb(args[i]); + } if (args[0]->myrank == 0 && !args[0]->no_rebuild) test_rebuild_wait(args, args_cnt); par_barrier(PAR_COMM_WORLD); for (i = 0; i < args_cnt; i++) { - if (args[i]->rebuild_post_cb) + if (args[i]->rebuild_post_cb) { + print_message( + "call rebuild_post_cb for exclude rebuilt pool " DF_UUID "\n", + DP_UUID(args[i]->pool.pool_uuid)); args[i]->rebuild_post_cb(args[i]); + } } return; @@ -179,6 +196,7 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, for (i = 0; i < rank_nr; i++) { int j; + const char *op_type_str = NULL; /* No concurrent drain/extend/reintegration are allowed, so * it has to reintegrate/extend one by one. @@ -187,15 +205,18 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, if (args[0]->myrank == 0) { switch (op_type) { case RB_OP_TYPE_REINT: + op_type_str = "reintegrate"; rebuild_reint_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1, kill); break; case RB_OP_TYPE_ADD: + op_type_str = "extend"; rebuild_extend_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1, args[i]->pool.pool_size); break; case RB_OP_TYPE_DRAIN: + op_type_str = "drain"; rebuild_drain_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1); break; @@ -208,20 +229,30 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, D_ASSERT(op_type != RB_OP_TYPE_RECLAIM); break; default: + op_type_str = "UNKNOWN"; break; } } par_barrier(PAR_COMM_WORLD); for (j = 0; j < args_cnt; j++) - if (args[j]->rebuild_cb) + if (args[j]->rebuild_cb) { + print_message("call rebuild_cb for %s rebuilding pool " DF_UUID + "\n", + op_type_str, DP_UUID(args[i]->pool.pool_uuid)); + args[j]->rebuild_cb(args[j]); + } if (args[0]->myrank == 0 && !args[0]->no_rebuild) test_rebuild_wait(args, args_cnt); for (j = 0; j < args_cnt; j++) { - if (args[j]->rebuild_post_cb) + if (args[j]->rebuild_post_cb) { + print_message("call rebuild_post_cb for %s rebuilt pool " DF_UUID + "\n", + op_type_str, DP_UUID(args[i]->pool.pool_uuid)); args[j]->rebuild_post_cb(args[j]); + } } } } @@ -970,8 +1001,8 @@ reintegrate_inflight_io(void *data) } ioreq_fini(&req); - sleep(12); print_message("sleep 12 seconds to wait for the stable epoch update.\n"); + sleep(12); if (arg->myrank == 0) daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0, NULL); @@ -1182,82 +1213,75 @@ get_killing_rank_by_oid(test_arg_t *arg, daos_obj_id_t oid, int data_nr, *ranks_num = idx; } -/* inject fault on leader engine to stop an in-progress rebuild */ -int -fi_rebuild_stop(void *data) +static int +rebuild_stop_with_dmg_internal(const char *cfg, const uuid_t uuid, const char *grp, bool force) { - test_arg_t *arg = data; - d_rank_t leader; - int rc; + int rc; - if (arg->myrank != 0) - return 0; + rc = dmg_pool_rebuild_stop(cfg, uuid, grp, force); + print_message("dmg pool rebuild stop " DF_UUID ", force=%d, rc=%d\n", DP_UUID(uuid), force, + rc); + assert_rc_equal(rc, 0); + return 0; +} - rc = test_get_leader(arg, &leader); - if (rc) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } +/* stop an in-progress rebuild with dmg pool rebuild stop command */ +int +rebuild_stop_with_dmg(void *data) +{ + test_arg_t *arg = data; - printf("%s(): pool " DF_UUIDF ": send ADMIN_STOP to leader rank %d\n", __FUNCTION__, - DP_UUID(arg->pool.pool_uuid), leader); - test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_STOP | DAOS_FAIL_ONCE); + print_message("wait for rebuild to start for pool " DF_UUID "\n", + DP_UUID(arg->pool.pool_uuid)); + test_rebuild_wait_to_start(&arg, 1); + sleep(4); - return 0; + return rebuild_stop_with_dmg_internal(arg->dmg_config, arg->pool.pool_uuid, arg->group, + false); } -/* inject fault on leader engine to receive stop command during op:Fail_reclaim */ +/* stop an in-progress rebuild with dmg pool rebuild stop command (force stop option) */ int -fi_rebuild_stop_reclaim(void *data) +rebuild_force_stop_with_dmg(void *data) { test_arg_t *arg = data; - d_rank_t leader; - int rc; - if (arg->myrank != 0) - return 0; + print_message("wait for rebuild to start for pool " DF_UUID "\n", + DP_UUID(arg->pool.pool_uuid)); + test_rebuild_wait_to_start(&arg, 1); + sleep(4); - rc = test_get_leader(arg, &leader); - if (rc) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } + return rebuild_stop_with_dmg_internal(arg->dmg_config, arg->pool.pool_uuid, arg->group, + true); +} - printf("%s(): pool " DF_UUIDF ": send ADMIN_STOP_RECLAIM to leader rank %d\n", __FUNCTION__, - DP_UUID(arg->pool.pool_uuid), leader); - test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_STOP_RECLAIM | DAOS_FAIL_ONCE); +/* start/reesume a stopped rebuild with dmg pool rebuild start command */ +int +rebuild_start_with_dmg(void *data) +{ + test_arg_t *arg = data; + int rc; + rc = dmg_pool_rebuild_start(arg->dmg_config, arg->pool.pool_uuid, arg->group); + print_message("dmg pool rebuild start " DF_UUID ", rc=%d\n", DP_UUID(arg->pool.pool_uuid), + rc); + assert_rc_equal(rc, 0); return 0; } -/* resume after a stopped rebuild, and optionally inject a fault to start a new rebuild */ int -fi_rebuild_resume_wait(void *data) +rebuild_resume_wait(void *data) { test_arg_t *arg = data; - struct daos_rebuild_status *rst = &arg->pool.pool_info.pi_rebuild_st; - + struct daos_rebuild_status *rst = &arg->pool.pool_info.pi_rebuild_st; bool skip_restart = false; - d_rank_t leader; int rc; - if (arg->myrank != 0) - return 0; - - if (arg->rebuild_cb == fi_rebuild_resume_wait && arg->rebuild_cb_arg) + if (arg->rebuild_cb == rebuild_resume_wait && arg->rebuild_cb_arg) skip_restart = *((bool *)arg->rebuild_cb_arg); - if (arg->rebuild_post_cb == fi_rebuild_resume_wait && arg->rebuild_post_cb_arg) + if (arg->rebuild_post_cb == rebuild_resume_wait && arg->rebuild_post_cb_arg) skip_restart = *((bool *)arg->rebuild_post_cb_arg); - rc = test_get_leader(arg, &leader); - if (rc) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } - /* Verify that the stop resulted in the correct rebuild status */ print_message("check: stopped rebuild rs_errno=%d (expect %d), rs_state=%d (expect %d)\n", rst->rs_errno, -DER_OP_CANCELED, rst->rs_state, DRS_NOT_STARTED); @@ -1268,10 +1292,8 @@ fi_rebuild_resume_wait(void *data) if (skip_restart) return 0; - /* Issue start fault injection to re-run the rebuild to successful completion */ - printf("%s(): pool " DF_UUIDF ": send ADMIN_START to leader rank %d\n", __FUNCTION__, - DP_UUID(arg->pool.pool_uuid), leader); - test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_START | DAOS_FAIL_ONCE); + rc = rebuild_start_with_dmg(data); + assert_rc_equal(rc, 0); /* Verify that the start resulted in a fully completed rebuild. * NB: loop to get past the immediate remnant state from the previous stop (OP_CANCELED) diff --git a/src/tests/suite/daos_rebuild_simple.c b/src/tests/suite/daos_rebuild_simple.c index 6b95dc09047..185deda338a 100644 --- a/src/tests/suite/daos_rebuild_simple.c +++ b/src/tests/suite/daos_rebuild_simple.c @@ -108,8 +108,8 @@ rebuild_dkeys(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -170,8 +170,8 @@ rebuild_akeys(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, tgt, false); arg->rebuild_cb = NULL; @@ -222,8 +222,8 @@ rebuild_indexes(void **state) /* Rebuild rank 1 */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -283,8 +283,8 @@ rebuild_snap_update_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -349,8 +349,8 @@ rebuild_snap_punch_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -409,8 +409,8 @@ rebuild_snap_update_keys(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -517,8 +517,8 @@ rebuild_snap_punch_keys(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -606,8 +606,8 @@ rebuild_snap_punch_empty(void **state) /* stop exclude rebuild, but skip the start, then directly reintegrate */ if (arg->interactive_rebuild) { bool skip_restart = true; - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; arg->rebuild_post_cb_arg = &skip_restart; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); @@ -655,8 +655,8 @@ rebuild_snap_punch_empty(void **state) /* from a stopped (not restarted) exclude, directly reintegrate (stop+start this rebuild) */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt); rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX); @@ -705,8 +705,8 @@ rebuild_multiple(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -763,8 +763,8 @@ rebuild_large_rec(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -803,8 +803,8 @@ rebuild_objects(void **state) rebuild_io(arg, oids, OBJ_NR); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -844,6 +844,12 @@ rebuild_sx_object_internal(void **state, daos_oclass_id_t oclass, if (!test_runable(arg, 4)) return; + if (arg->interactive_rebuild && !wait_rebuild) { + print_message("SKIP due to interactive_rebuild enabled, but not tested here\n"); + skip(); + return; + } + oid = daos_test_oid_gen(arg->coh, oclass, 0, 0, arg->myrank); ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); print_message("insert 100 dkeys\n"); @@ -874,30 +880,35 @@ rebuild_sx_object_internal(void **state, daos_oclass_id_t oclass, get_killing_rank_by_oid(arg, oid, 1, 0, &rank, &rank_nr); /** exclude the target of this obj's replicas */ - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool exclude rank %u " DF_UUID "\n", rank, DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_exclude(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until exclude rebuild done */ if (wait_rebuild) { test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); } - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool reintegrate rank %u " DF_UUID "\n", rank, + DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_reintegrate(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until reintegration rebuild is done */ if (wait_rebuild) { test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); } print_message("lookup 100 dkeys\n"); @@ -978,27 +989,33 @@ rebuild_large_object(void **state) ioreq_fini(&req); } - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool exclude rank %u " DF_UUID "\n", rank, DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_exclude(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until exclude rebuild done */ test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool reintegrate rank %u " DF_UUID "\n", rank, + DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_reintegrate(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) { + rebuild_stop_with_dmg(arg); + } + /* wait until reintegration rebuild is done */ test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); print_message("success\n"); } @@ -1094,8 +1111,8 @@ rebuild_large_snap(void **state) /* stop and resume the exclude rebuild */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); ioreq_fini(&req); @@ -1142,8 +1159,8 @@ rebuild_full_shards(void **state) /* rebuild and reintegration to use full shards */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } /* stop and resume the first exclude rebuild (but not the second one) */ rebuild_single_pool_target(arg, 0, -1, false); @@ -1153,8 +1170,8 @@ rebuild_full_shards(void **state) /* stop and resume the first reintegrate rebuild (but not the second one) */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } reintegrate_single_pool_target(arg, 0, -1); arg->rebuild_cb = NULL; @@ -1196,8 +1213,8 @@ rebuild_punch_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], -1, false); arg->rebuild_cb = NULL; @@ -1253,8 +1270,8 @@ rebuild_multiple_group(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1264,10 +1281,6 @@ rebuild_multiple_group(void **state) if (rc != 0) assert_rc_equal(rc, -DER_NOSYS); - if (arg->interactive_rebuild) { - arg->rebuild_cb = NULL; - arg->rebuild_post_cb = NULL; - } reintegrate_with_inflight_io(arg, &oid, kill_rank, -1); rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX); @@ -1311,8 +1324,8 @@ rebuild_with_large_offset(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1363,8 +1376,8 @@ rebuild_with_large_key(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1432,8 +1445,8 @@ rebuild_with_dfs_open_create_punch(void **state) dfs_obj2id(dir, &oid); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rank = get_rank_by_oid_shard(arg, oid, 0); rebuild_single_pool_rank(arg, rank, false); @@ -1487,6 +1500,40 @@ rebuild_wait_reset_fail_cb(void *data) return 0; } +static int +rebuild_wait_error_reset_fail_cb(void *data) +{ + test_arg_t *arg = data; + int rc; + + print_message("wait until rebuild errors (and starts Fail_reclaim)\n"); + test_rebuild_wait_to_error(&arg, 1); + print_message("check rebuild errored, rs_errno=%d (expecting -DER_IO=%d)\n", + arg->pool.pool_info.pi_rebuild_st.rs_errno, -DER_IO); + assert_int_equal(arg->pool.pool_info.pi_rebuild_st.rs_errno, -DER_IO); + print_message("rebuild error code check passed\n"); + + print_message("clearing fault injection on all engines\n"); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0, NULL); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 0, 0, NULL); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_NUM, 0, 0, NULL); + + /* Give time for transition from op:Rebuild into op:Fail_reclaim */ + sleep(2); + + print_message( + "send rebuild stop --force request during first/only Fail_reclaim operation\n"); + rc = rebuild_force_stop_with_dmg(data); + if (rc != 0) + print_message("rebuild_force_stop_with_dmg failed, rc=%d\n", rc); + + print_message("wait for rebuild to be stopped\n"); + test_rebuild_wait(&arg, 1); + /* Verifying rs_state/rs_errno will happen in post_cb rebuild_resume_wait() */ + + return rc; +} + static void rebuild_many_objects_with_failure(void **state) { @@ -1516,46 +1563,25 @@ rebuild_many_objects_with_failure(void **state) /* Inject faults on engines. Special handling for interactive_rebuild case */ if (arg->myrank == 0) { - if (!arg->interactive_rebuild) { - /* All engines DAOS_REBUILD_OBJ_FAIL */ - print_message("inject fault DAOS_REBUILD_OBJ_FAIL on all engines\n"); - daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, - DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); - daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 50, 0, NULL); - } else { - d_rank_t leader; - - /* interactive_rebiuld: DAOS_REBUILD_OBJ fail on all engines except PS - * leader; DAOS_REBUILD_ADMIN_STOP_RECLAIM on the PS leader engine - */ - - rc = test_get_leader(arg, &leader); - if (rc != 0) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } - - for (i = 0; i < arg->srv_nnodes; i++) { - if (i == leader) - fi_rebuild_stop_reclaim(arg); - else { - print_message("inject DAOS_REBUILD_OBJ_FAIL on non-leader " - "engine %u\n", - i); - daos_debug_set_params( - arg->group, i, DMG_KEY_FAIL_LOC, - DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); - daos_debug_set_params(arg->group, i, DMG_KEY_FAIL_VALUE, 50, - 0, NULL); - } - } - } + print_message("inject fault DAOS_REBUILD_OBJ_FAIL on all engines\n"); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, + DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 50, 0, NULL); } - arg->rebuild_cb = rebuild_wait_reset_fail_cb; - if (arg->interactive_rebuild) - arg->rebuild_post_cb = fi_rebuild_resume_wait; + /* For interactive rebuild, we need: + * 1. trigger rebuild (which will fail), query pool reubild state until op:Rebuild fails + * and op:Fail_reclaim begins. See test_rebuild_wait_to_error(). + * 2. Then, while rebuild is in op:Fail_reclaim, issue dmg system stop to test that you + * can't stop during Fail_reclaim (though the command will take effect by not retrying + * rebuild). + */ + if (arg->interactive_rebuild) { + arg->rebuild_cb = rebuild_wait_error_reset_fail_cb; + arg->rebuild_post_cb = rebuild_resume_wait; + } else { + arg->rebuild_cb = rebuild_wait_reset_fail_cb; + } rebuild_single_pool_target(arg, 3, -1, false); for (i = 0; i < 8000; i++) { diff --git a/src/tests/suite/daos_test.h b/src/tests/suite/daos_test.h index 97a56bdec7d..b6f15f06fbb 100644 --- a/src/tests/suite/daos_test.h +++ b/src/tests/suite/daos_test.h @@ -397,6 +397,10 @@ int test_pool_get_info(test_arg_t *arg, daos_pool_info_t *pinfo, d_rank_list_t * int test_get_leader(test_arg_t *arg, d_rank_t *rank); bool test_rebuild_query(test_arg_t **args, int args_cnt); void test_rebuild_wait(test_arg_t **args, int args_cnt); +void +test_rebuild_wait_to_start(test_arg_t **args, int args_cnt); +void + test_rebuild_wait_to_error(test_arg_t **args, int args_cnt); int daos_pool_set_prop(const uuid_t pool_uuid, const char *name, const char *value); @@ -456,24 +460,34 @@ void rebuild_add_back_tgts(test_arg_t *arg, d_rank_t failed_rank, int rebuild_pool_disconnect_internal(void *data); int rebuild_pool_connect_internal(void *data); - -int rebuild_sub_setup(void **state); -int rebuild_sub_rf1_setup(void **state); -int rebuild_sub_rf0_setup(void **state); -int rebuild_sub_teardown(void **state); -int rebuild_small_sub_setup(void **state); -int rebuild_small_sub_rf1_setup(void **state); -int rebuild_small_sub_rf0_setup(void **state); -int rebuild_sub_3nodes_rf0_setup(void **state); -int rebuild_sub_6nodes_rf1_setup(void **state); -int rebuild_sub_setup_common(void **state, daos_size_t pool_size, int node_nr, uint32_t rf); - int -fi_rebuild_stop(void *data); +rebuild_sub_setup(void **state); +int +rebuild_sub_rf1_setup(void **state); +int +rebuild_sub_rf0_setup(void **state); +int +rebuild_sub_teardown(void **state); +int +rebuild_small_sub_setup(void **state); +int +rebuild_small_sub_rf1_setup(void **state); +int +rebuild_small_sub_rf0_setup(void **state); +int +rebuild_sub_3nodes_rf0_setup(void **state); +int +rebuild_sub_6nodes_rf1_setup(void **state); +int +rebuild_sub_setup_common(void **state, daos_size_t pool_size, int node_nr, uint32_t rf); +int +rebuild_stop_with_dmg(void *data); +int +rebuild_force_stop_with_dmg(void *data); int -fi_rebuild_stop_reclaim(void *data); +rebuild_start_with_dmg(void *data); int - fi_rebuild_resume_wait(void *data); + rebuild_resume_wait(void *data); int get_server_config(char *host, char *server_config_file); int get_log_file(char *host, char *server_config_file, diff --git a/src/tests/suite/daos_test_common.c b/src/tests/suite/daos_test_common.c index 2f6b3fd3691..21bd5a7fb7d 100644 --- a/src/tests/suite/daos_test_common.c +++ b/src/tests/suite/daos_test_common.c @@ -791,7 +791,61 @@ test_pool_get_info(test_arg_t *arg, daos_pool_info_t *pinfo, d_rank_list_t **eng } static bool -rebuild_pool_wait(test_arg_t *arg) +rebuild_pool_started(test_arg_t *arg) +{ + daos_pool_info_t pinfo = {0}; + struct daos_rebuild_status *rst; + int rc; + + pinfo.pi_bits = DPI_REBUILD_STATUS; + rc = test_pool_get_info(arg, &pinfo, NULL /* engine_ranks */); + rst = &pinfo.pi_rebuild_st; + + if (rc != 0) { + print_message("pool query for rebuild status failed, rc=%d, pool " DF_UUIDF "\n", + rc, DP_UUID(arg->pool.pool_uuid)); + return false; + } else { + bool started = (rst->rs_state == DRS_IN_PROGRESS); + print_message("rebuild for pool " DF_UUIDF "has %sstarted\n", + DP_UUID(arg->pool.pool_uuid), started ? "" : "not yet "); + return started; + } +} + +static bool +rebuild_pool_erroring(test_arg_t *arg) +{ + daos_pool_info_t pinfo = {0}; + struct daos_rebuild_status *rst; + int rc; + + pinfo.pi_bits = DPI_REBUILD_STATUS; + rc = test_pool_get_info(arg, &pinfo, NULL /* engine_ranks */); + rst = &pinfo.pi_rebuild_st; + + if (rc != 0) { + print_message("pool query for rebuild status failed, rc=%d, pool " DF_UUIDF "\n", + rc, DP_UUID(arg->pool.pool_uuid)); + return false; + } else { + bool started = (rst->rs_state == DRS_IN_PROGRESS); + bool erroring = started && (rst->rs_errno != 0); + + print_message("rebuild for pool " DF_UUIDF "has %sstarted, rs_errno=%d\n", + DP_UUID(arg->pool.pool_uuid), started ? "" : "not yet ", + rst->rs_errno); + + /* save final pool query info to be able to inspect rebuild status */ + if (erroring) + memcpy(&arg->pool.pool_info, &pinfo, sizeof(pinfo)); + + return erroring; + } +} + +static bool +rebuild_pool_done(test_arg_t *arg) { daos_pool_info_t pinfo = {0}; struct daos_rebuild_status *rst; @@ -862,6 +916,56 @@ test_get_last_svr_rank(test_arg_t *arg) return arg->srv_nnodes - disable_nodes - 1; } +bool +test_rebuild_started(test_arg_t **args, int args_cnt) +{ + bool all_started = true; + int i; + + for (i = 0; i < args_cnt; i++) { + bool started = true; + + if (!args[i]->pool.destroyed) + started = rebuild_pool_started(args[i]); + + if (!started) + all_started = false; + } + return all_started; +} + +void +test_rebuild_wait_to_start(test_arg_t **args, int args_cnt) +{ + while (!test_rebuild_started(args, args_cnt)) + sleep(2); +} + +bool +test_rebuild_erroring(test_arg_t **args, int args_cnt) +{ + bool all_erroring = true; + int i; + + for (i = 0; i < args_cnt; i++) { + bool erroring = true; + + if (!args[i]->pool.destroyed) + erroring = rebuild_pool_erroring(args[i]); + + if (!erroring) + all_erroring = false; + } + return all_erroring; +} + +void +test_rebuild_wait_to_error(test_arg_t **args, int args_cnt) +{ + while (!test_rebuild_erroring(args, args_cnt)) + sleep(1); +} + bool test_rebuild_query(test_arg_t **args, int args_cnt) { @@ -872,7 +976,7 @@ test_rebuild_query(test_arg_t **args, int args_cnt) bool done = true; if (!args[i]->pool.destroyed) - done = rebuild_pool_wait(args[i]); + done = rebuild_pool_done(args[i]); if (!done) all_done = false; From 1fb93d7048829ab2af079dd4c69715b8f19e6761 Mon Sep 17 00:00:00 2001 From: Kenneth Cain Date: Wed, 24 Sep 2025 12:18:52 -0400 Subject: [PATCH 4/4] Fix this PR's segfault in printfs added in rebuild_targets() Increase sleep time by one more second in rebuild test functions rebuild_stop_with_dmg() and rebuild_force_stop_with_dmg(), to better ensure stop arrives during Fail_reclaim in the test (rebuild_many_objects_with_failure). Test-tag: pr test_daos_rebuild_simple_interactive Allow-unstable-test: true Signed-off-by: Kenneth Cain --- src/tests/suite/daos_rebuild_common.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/tests/suite/daos_rebuild_common.c b/src/tests/suite/daos_rebuild_common.c index 8f1bc7eb944..959cf042278 100644 --- a/src/tests/suite/daos_rebuild_common.c +++ b/src/tests/suite/daos_rebuild_common.c @@ -238,7 +238,7 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, if (args[j]->rebuild_cb) { print_message("call rebuild_cb for %s rebuilding pool " DF_UUID "\n", - op_type_str, DP_UUID(args[i]->pool.pool_uuid)); + op_type_str, DP_UUID(args[j]->pool.pool_uuid)); args[j]->rebuild_cb(args[j]); } @@ -250,7 +250,7 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, if (args[j]->rebuild_post_cb) { print_message("call rebuild_post_cb for %s rebuilt pool " DF_UUID "\n", - op_type_str, DP_UUID(args[i]->pool.pool_uuid)); + op_type_str, DP_UUID(args[j]->pool.pool_uuid)); args[j]->rebuild_post_cb(args[j]); } } @@ -996,9 +996,8 @@ reintegrate_inflight_io(void *data) SMALL_SINGLE_VALUE_SIZE, DAOS_TX_NONE, &req); sprintf(key, "d_inflight_single_large_%d", i); - insert_single(key, "a_key", 0, single_data, - LARGE_SINGLE_VALUE_SIZE, DAOS_TX_NONE, &req); - + insert_single(key, "a_key", 0, single_data, LARGE_SINGLE_VALUE_SIZE, DAOS_TX_NONE, + &req); } ioreq_fini(&req); print_message("sleep 12 seconds to wait for the stable epoch update.\n"); @@ -1234,7 +1233,7 @@ rebuild_stop_with_dmg(void *data) print_message("wait for rebuild to start for pool " DF_UUID "\n", DP_UUID(arg->pool.pool_uuid)); test_rebuild_wait_to_start(&arg, 1); - sleep(4); + sleep(5); return rebuild_stop_with_dmg_internal(arg->dmg_config, arg->pool.pool_uuid, arg->group, false); @@ -1249,7 +1248,7 @@ rebuild_force_stop_with_dmg(void *data) print_message("wait for rebuild to start for pool " DF_UUID "\n", DP_UUID(arg->pool.pool_uuid)); test_rebuild_wait_to_start(&arg, 1); - sleep(4); + sleep(5); return rebuild_stop_with_dmg_internal(arg->dmg_config, arg->pool.pool_uuid, arg->group, true);