diff --git a/src/common/tests_dmg_helpers.c b/src/common/tests_dmg_helpers.c index dcf1b274116..08825366133 100644 --- a/src/common/tests_dmg_helpers.c +++ b/src/common/tests_dmg_helpers.c @@ -1206,6 +1206,80 @@ dmg_pool_list(const char *dmg_config_file, const char *group, return rc; } +int +dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force) +{ + char uuid_str[DAOS_UUID_STR_SIZE]; + int argcount = 0; + char **args = NULL; + struct json_object *dmg_out = NULL; + int rc = 0; + + uuid_unparse_lower(uuid, uuid_str); + args = cmd_push_arg(args, &argcount, "%s ", uuid_str); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + + if (grp != NULL) { + args = cmd_push_arg(args, &argcount, "--sys=%s ", grp); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + if (force) { + args = cmd_push_arg(args, &argcount, "--force"); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + rc = daos_dmg_json_pipe("pool rebuild stop", dmg_config_file, args, argcount, &dmg_out); + if (rc != 0) { + D_ERROR("dmg pool rebuild stop failed\n"); + goto out_json; + } + +out_json: + if (dmg_out != NULL) + json_object_put(dmg_out); + cmd_free_args(args, argcount); +out: + return rc; +} + +int +dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp) +{ + char uuid_str[DAOS_UUID_STR_SIZE]; + int argcount = 0; + char **args = NULL; + struct json_object *dmg_out = NULL; + int rc = 0; + + uuid_unparse_lower(uuid, uuid_str); + args = cmd_push_arg(args, &argcount, "%s ", uuid_str); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + + if (grp != NULL) { + args = cmd_push_arg(args, &argcount, "--sys=%s ", grp); + if (args == NULL) + D_GOTO(out, rc = -DER_NOMEM); + } + + rc = daos_dmg_json_pipe("pool rebuild start", dmg_config_file, args, argcount, &dmg_out); + if (rc != 0) { + D_ERROR("dmg pool rebuild start failed\n"); + goto out_json; + } + +out_json: + if (dmg_out != NULL) + json_object_put(dmg_out); + cmd_free_args(args, argcount); +out: + return rc; +} + static int parse_device_info(struct json_object *smd_dev, device_list *devices, char *host, int dev_length, int *disks) diff --git a/src/include/daos/common.h b/src/include/daos/common.h index 7999879193a..6c1d24fd1ac 100644 --- a/src/include/daos/common.h +++ b/src/include/daos/common.h @@ -789,9 +789,6 @@ enum { #define DAOS_RDB_SKIP_APPENDENTRIES_FAIL (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1a) #define DAOS_FORCE_REFRESH_POOL_MAP (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1b) -#define DAOS_REBUILD_ADMIN_STOP (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1c) -#define DAOS_REBUILD_ADMIN_START (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1d) - #define DAOS_FORCE_CAPA_FETCH (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1e) #define DAOS_FORCE_PROP_VERIFY (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1f) diff --git a/src/include/daos/tests_lib.h b/src/include/daos/tests_lib.h index 8c9ed5bf1f9..72438598297 100644 --- a/src/include/daos/tests_lib.h +++ b/src/include/daos/tests_lib.h @@ -484,6 +484,25 @@ dmg_pool_set_prop(const char *dmg_config_file, int dmg_pool_get_prop(const char *dmg_config_file, const char *label, const uuid_t uuid, const char *name, char **value); +/** + * Interactively stop a pool's currently-running rebuild. + * \param dmg_config_file [IN] DMG config file. + * \param uuid [IN] UUID of the pool. + * \param grp [IN] Process set name of the DAOS servers managing the pool. + * \param force [IN] forcibly stop a rebuild that is failing. + */ +int +dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force); + +/** + * Interactively start/resume a pool's rebuilding. + * \param dmg_config_file [IN] DMG config file. + * \param uuid [IN] UUID of the pool. + * \param grp [IN] Process set name of the DAOS servers managing the pool. + */ +int + dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp); + /** * List all disks in the specified DAOS system. * diff --git a/src/include/daos_srv/rebuild.h b/src/include/daos_srv/rebuild.h index d5714314bbd..553f1a6ff45 100644 --- a/src/include/daos_srv/rebuild.h +++ b/src/include/daos_srv/rebuild.h @@ -87,7 +87,7 @@ int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t stable_eph, uint32_t layout_version, struct pool_target_id_list *tgts, daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op, - uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec); + uint32_t retry_map_ver, bool stop_admin, void *cur_taskp, uint64_t delay_sec); void ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank); int ds_rebuild_query(uuid_t pool_uuid, struct daos_rebuild_status *status); diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 276b311c4a2..5441d52cf00 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -6410,7 +6410,7 @@ pool_check_upgrade_object_layout(struct rdb_tx *tx, struct pool_svc *svc, rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, upgrade_eph, DS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE, RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */, - false /* stop_admin */, 0); + false /* stop_admin */, NULL /* cur_taskp */, 0); if (rc == 0) *scheduled_layout_upgrade = true; } @@ -7758,7 +7758,8 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank, if (tgt_map_ver != 0) { rc = ds_rebuild_schedule(svc->ps_pool, tgt_map_ver, rebuild_eph, 0, &target_list, RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay); + 0 /* retry_map_ver */, false /* stop_admin */, + NULL /* cur_taskp */, delay); if (rc != 0) { D_ERROR("rebuild fails rc: "DF_RC"\n", DP_RC(rc)); D_GOTO(out, rc); diff --git a/src/rebuild/rebuild_internal.h b/src/rebuild/rebuild_internal.h index c05af6c02e9..14564dbf746 100644 --- a/src/rebuild/rebuild_internal.h +++ b/src/rebuild/rebuild_internal.h @@ -168,6 +168,11 @@ struct rebuild_global_pool_tracker { uint32_t rgt_opc; unsigned int rgt_abort : 1, /* abort: kill rebuild */ rgt_init_scan : 1, rgt_stop_admin : 1; /* stop: admin has asked to kill rebuild */ + + uint32_t rgt_num_op_rb; /* count of op:Rebuild attempts */ + uint32_t rgt_num_op_freclaim; /* count of op:Fail_reclaim attempts */ + uint32_t rgt_num_op_rb_fail; /* count of op:Rebuild failures */ + uint32_t rgt_num_op_freclaim_fail; /* count of failed op:Fail_reclaim (not good) */ }; /* Structure on raft replica nodes to serve completed rebuild status querying */ @@ -254,6 +259,14 @@ struct rebuild_task { * Then, on fail_reclaim finish, the pool rebuild state will be set to idle (NOT_STARTED). */ bool dst_stop_admin; + + /* Track how many tries for certain daos_rebuild_opc_t */ + uint32_t dst_num_op_rb; /* count of tries to run rebuild */ + uint32_t dst_num_op_reclaim; + uint32_t dst_num_op_freclaim; + uint32_t dst_num_op_upgrade; + uint32_t dst_num_op_rb_fail; /* count of rebuild failures */ + uint32_t dst_num_op_freclaim_fail; /* count of Fail_recliam failures */ }; /* Per pool structure in TLS to check pool rebuild status diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index 415b7df791f..7a51396ddf8 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -874,8 +874,17 @@ enum { RB_BCAST_QUERY, }; -/* for interactive mode rebuild fault-injection testing */ -static uuid_t test_stop_start_puuid; +static bool +rebuild_is_stoppable(struct rebuild_global_pool_tracker *rgt, bool force) +{ + if ((rgt->rgt_opc == RB_OP_REBUILD) || (rgt->rgt_opc == RB_OP_UPGRADE)) + return true; + + if ((rgt->rgt_opc == RB_OP_FAIL_RECLAIM) && force && (rgt->rgt_num_op_freclaim_fail > 0)) + return true; + + return false; +} /* administrator (via dmg command) asks to stop the currently-running rebuild for pool */ int @@ -893,8 +902,7 @@ ds_rebuild_admin_stop(struct ds_pool *pool, uint32_t force) } /* admin stop command does not terminate reclaim/fail_reclaim jobs (unless forced) */ - if (((rgt->rgt_opc == RB_OP_REBUILD) || (rgt->rgt_opc == RB_OP_UPGRADE)) || - ((rgt->rgt_opc == RB_OP_FAIL_RECLAIM) && force)) { + if (rebuild_is_stoppable(rgt, force)) { D_INFO(DF_RB ": stopping rebuild force=%u opc %u(%s)\n", DP_RB_RGT(rgt), force, rgt->rgt_opc, RB_OP_STR(rgt->rgt_opc)); rgt->rgt_abort = 1; @@ -925,9 +933,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, unsigned int total; struct sched_req_attr attr = {0}; d_rank_t myrank; - unsigned int nsleeps = 0; - bool fi_admin_stop = false; - bool fi_admin_stop_frc = false; int rc; rc = crt_group_size(pool->sp_group, &total); @@ -952,19 +957,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, bool rebuild_abort = false; int i; - if (!fi_admin_stop && (op == RB_OP_REBUILD)) - fi_admin_stop = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP); - if (!fi_admin_stop_frc && (op == RB_OP_FAIL_RECLAIM)) - fi_admin_stop_frc = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_STOP_RECLAIM); - - if ((fi_admin_stop || fi_admin_stop_frc) && (nsleeps > 0)) { - uuid_copy(test_stop_start_puuid, rgt->rgt_pool_uuid); - - /* What the real RPC handler will call. We will exit below due to rgt_abort - */ - ds_pool_rebuild_stop(test_stop_start_puuid, 0 /* force */, NULL /* hint */); - } - ABT_rwlock_rdlock(pool->sp_lock); rc = map_ranks_init(pool->sp_map, PO_COMP_ST_UP | PO_COMP_ST_DOWN | @@ -1060,8 +1052,6 @@ rebuild_leader_status_check(struct ds_pool *pool, uint32_t op, sleep: update_and_warn_for_slow_engines(rgt); sched_req_sleep(rgt->rgt_ult, RBLD_CHECK_INTV); - if (fi_admin_stop || fi_admin_stop_frc) - nsleeps++; } sched_req_put(rgt->rgt_ult); @@ -1665,6 +1655,10 @@ rebuild_leader_start(struct ds_pool *pool, struct rebuild_task *task, return rc; D_ASSERT(*p_rgt != NULL); + (*p_rgt)->rgt_num_op_rb = task->dst_num_op_rb; + (*p_rgt)->rgt_num_op_freclaim = task->dst_num_op_freclaim; + (*p_rgt)->rgt_num_op_rb_fail = task->dst_num_op_rb_fail; + (*p_rgt)->rgt_num_op_freclaim_fail = task->dst_num_op_freclaim_fail; D_INFO(DF_RB "\n", DP_RB_RGT(*p_rgt)); /* broadcast scan RPC to all targets */ @@ -1778,10 +1772,10 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, return 0; } - rc = ds_rebuild_schedule(pool, task->dst_map_ver, task->dst_reclaim_eph, - task->dst_new_layout_version, &task->dst_tgts, - task->dst_rebuild_op, task->dst_retry_rebuild_op, - task->dst_retry_map_ver, task->dst_stop_admin, delay_sec); + rc = ds_rebuild_schedule( + pool, task->dst_map_ver, task->dst_reclaim_eph, task->dst_new_layout_version, + &task->dst_tgts, task->dst_rebuild_op, task->dst_retry_rebuild_op, + task->dst_retry_map_ver, task->dst_stop_admin, NULL /* cur_taskp */, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_UUID ": reschedule not-started %u(%s)", DP_UUID(task->dst_pool_uuid), task->dst_rebuild_op, RB_OP_STR(task->dst_rebuild_op)); @@ -1815,7 +1809,7 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, task->dst_new_layout_version, &task->dst_tgts, task->dst_rebuild_op, task->dst_retry_rebuild_op, task->dst_retry_map_ver, task->dst_stop_admin, - delay_sec); + task, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule retry %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), task->dst_rebuild_op, @@ -1835,7 +1829,7 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, pool, task->dst_reclaim_ver - 1 /* map_ver */, rgt->rgt_stable_epoch, task->dst_new_layout_version, &task->dst_tgts, RB_OP_FAIL_RECLAIM, retry_opc /* retry_rebuild_op */, task->dst_map_ver /* retry_map_ver */, - rgt->rgt_stop_admin, delay_sec); + rgt->rgt_stop_admin, task, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), RB_OP_FAIL_RECLAIM, @@ -1855,10 +1849,10 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, if (retry_opc == RB_OP_NONE) D_GOTO(complete, rc); - rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_stable_epoch, - task->dst_new_layout_version, &task->dst_tgts, - retry_opc /* rebuild_op*/, 0 /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay_sec); + rc = ds_rebuild_schedule( + pool, task->dst_map_ver, rgt->rgt_stable_epoch, task->dst_new_layout_version, + &task->dst_tgts, retry_opc /* rebuild_op*/, 0 /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, task, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule retry of original %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), retry_opc, @@ -1868,10 +1862,10 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, * (exclude/drain/reintegrate/extend/upgrade). */ rgt->rgt_status.rs_state = DRS_IN_PROGRESS; - rc = ds_rebuild_schedule(pool, task->dst_map_ver, rgt->rgt_reclaim_epoch, - task->dst_new_layout_version, &task->dst_tgts, - RB_OP_RECLAIM, RB_OP_NONE /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay_sec); + rc = ds_rebuild_schedule( + pool, task->dst_map_ver, rgt->rgt_reclaim_epoch, task->dst_new_layout_version, + &task->dst_tgts, RB_OP_RECLAIM, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, task, delay_sec); DL_CDEBUG(rc, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), RB_OP_RECLAIM, RB_OP_STR(RB_OP_RECLAIM)); @@ -1910,7 +1904,7 @@ rebuild_task_complete_schedule(struct rebuild_task *task, struct ds_pool *pool, task->dst_new_layout_version, &task->dst_tgts, task->dst_retry_rebuild_op, RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */, - false /* stop_admin */, delay_sec); + false /* stop_admin */, task, delay_sec); DL_CDEBUG(rc1, DLOG_ERR, DLOG_INFO, rc, DF_RB ": errno " DF_RC ", schedule retry %u(%s)", DP_RB_RGT(rgt), DP_RC(rgt->rgt_status.rs_errno), task->dst_retry_rebuild_op, @@ -2130,25 +2124,13 @@ rebuild_ults(void *arg) while (DAOS_FAIL_CHECK(DAOS_REBUILD_HANG)) ABT_thread_yield(); - /* Stay alive for fault-injection possibility (need a ULT for DAOS_REBUILD_ADMIN_START) */ - while (true) { - bool fi_admin_start = false; - + while (!d_list_empty(&rebuild_gst.rg_queue_list) || + !d_list_empty(&rebuild_gst.rg_running_list)) { if (rebuild_gst.rg_abort) { D_INFO("abort rebuilds\n"); break; } - fi_admin_start = DAOS_FAIL_CHECK(DAOS_REBUILD_ADMIN_START); - if (fi_admin_start) { - if (!uuid_is_null(test_stop_start_puuid)) { - D_INFO("fi_admin_start=%d, apply to pool " DF_UUID "\n", - fi_admin_start, DP_UUID(test_stop_start_puuid)); - ds_pool_rebuild_start(test_stop_start_puuid, NULL /* hint */); - uuid_clear(test_stop_start_puuid); - } - } - if (d_list_empty(&rebuild_gst.rg_queue_list) || rebuild_gst.rg_inflight >= REBUILD_MAX_INFLIGHT) { D_INFO("in-flight rebuild %u - going to sleep for 5 seconds\n", @@ -2352,8 +2334,9 @@ int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t reclaim_eph, uint32_t layout_version, struct pool_target_id_list *tgts, daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op, - uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec) + uint32_t retry_map_ver, bool stop_admin, void *cur_taskp, uint64_t delay_sec) { + struct rebuild_task *cur_task = cur_taskp; struct rebuild_task *new_task; struct rebuild_task *task; d_list_t *inserted_pos; @@ -2408,6 +2391,60 @@ ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t reclaim new_task->dst_retry_rebuild_op = RB_OP_NONE; new_task->dst_stop_admin = false; } + + /* update operation counts based on this (cur_task->dst_rebuild_op) -> new (rebuild_op), + * e.g.: op:Rebuild -> op:Reclaim or op:Fail_reclaim op:Reclaim -> op:Reclaim (unexpected, + * not tracked / Reclaim failure) op:Rebuild -> op:Rebuild (retry original rebuild / + * scheduled when original Rebuild fails) op:Fail_reclaim -> op:Rebuild (retry original + * rebuild / scheduled after Fail_reclaim done) op:Fail_reclaim -> op:Fail_reclaim + * (unexpected and tracked / Fail_reclaim failure) + */ + /* TODO: does upgrade ever happen with cur_task != NULL? */ + new_task->dst_num_op_rb = cur_task ? cur_task->dst_num_op_rb : 0; + new_task->dst_num_op_reclaim = cur_task ? cur_task->dst_num_op_reclaim : 0; + new_task->dst_num_op_freclaim = cur_task ? cur_task->dst_num_op_freclaim : 0; + new_task->dst_num_op_upgrade = cur_task ? cur_task->dst_num_op_upgrade : 0; + new_task->dst_num_op_rb_fail = cur_task ? cur_task->dst_num_op_rb_fail : 0; + new_task->dst_num_op_freclaim_fail = cur_task ? cur_task->dst_num_op_freclaim_fail : 0; + + switch (rebuild_op) { + case RB_OP_REBUILD: + new_task->dst_num_op_rb++; + break; + case RB_OP_RECLAIM: + D_ASSERT(cur_task != NULL); + new_task->dst_num_op_reclaim++; + break; + case RB_OP_FAIL_RECLAIM: + D_ASSERT(cur_task != NULL); + new_task->dst_num_op_freclaim++; + /* NB: update fail count in cur_task in case caller may be scheduling both + * Fail_reclaim and retry of op:Rebuild at the same time (i.e., while processing + * completion of the original/failed Rebuild). + */ + if (cur_task->dst_rebuild_op == RB_OP_REBUILD) + new_task->dst_num_op_rb_fail = ++cur_task->dst_num_op_rb_fail; + + /* Fail_reclaim failing repeatedly */ + if (cur_task->dst_rebuild_op == RB_OP_FAIL_RECLAIM) + new_task->dst_num_op_freclaim_fail++; + break; + case RB_OP_UPGRADE: + new_task->dst_num_op_upgrade++; + break; + default: + break; + } + + D_INFO(DF_UUID + ": %s%s%s (ver=%u), counts (rb/reclaim/freclaim/upgrade/rb_fail/freclaim_fail)" + "= %u/%u/%u/%u/%u/%u\n", + DP_UUID(pool->sp_uuid), cur_task ? RB_OP_STR(cur_task->dst_rebuild_op) : "", + cur_task ? "->" : "", RB_OP_STR(new_task->dst_rebuild_op), new_task->dst_map_ver, + new_task->dst_num_op_rb, new_task->dst_num_op_reclaim, new_task->dst_num_op_freclaim, + new_task->dst_num_op_upgrade, new_task->dst_num_op_rb_fail, + new_task->dst_num_op_freclaim_fail); + uuid_copy(new_task->dst_pool_uuid, pool->sp_uuid); D_INIT_LIST_HEAD(&new_task->dst_list); @@ -2494,16 +2531,18 @@ regenerate_task_internal(struct ds_pool *pool, struct pool_target *tgts, id_list.pti_number = 1; if (tgt->ta_comp.co_status & (PO_COMP_ST_DOWN | PO_COMP_ST_DRAIN)) { - rc = ds_rebuild_schedule( - pool, tgt->ta_comp.co_fseq, current_eph == 0 ? eph : current_eph, 0, - &id_list, RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay); + rc = ds_rebuild_schedule(pool, tgt->ta_comp.co_fseq, + current_eph == 0 ? eph : current_eph, 0, &id_list, + RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, + NULL /* cur_taskp */, delay); } else { D_ASSERT(tgt->ta_comp.co_status == PO_COMP_ST_UP); - rc = ds_rebuild_schedule( - pool, tgt->ta_comp.co_in_ver, current_eph == 0 ? eph : current_eph, 0, - &id_list, RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, - 0 /* retry_map_ver */, false /* stop_admin */, delay); + rc = ds_rebuild_schedule(pool, tgt->ta_comp.co_in_ver, + current_eph == 0 ? eph : current_eph, 0, &id_list, + RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */, + 0 /* retry_map_ver */, false /* stop_admin */, + NULL /* cur_taskp */, delay); } if (rc) { D_ERROR(DF_UUID" schedule ver %d failed: "DF_RC"\n", @@ -3105,8 +3144,6 @@ init(void) { int rc; - uuid_clear(test_stop_start_puuid); - D_INIT_LIST_HEAD(&rebuild_gst.rg_tgt_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_global_tracker_list); D_INIT_LIST_HEAD(&rebuild_gst.rg_completed_list); diff --git a/src/tests/suite/daos_rebuild_common.c b/src/tests/suite/daos_rebuild_common.c index 5ecd1e0439d..959cf042278 100644 --- a/src/tests/suite/daos_rebuild_common.c +++ b/src/tests/suite/daos_rebuild_common.c @@ -62,6 +62,8 @@ rebuild_exclude_tgt(test_arg_t **args, int arg_cnt, d_rank_t rank, for (i = 0; i < arg_cnt; i++) { rc = dmg_pool_exclude(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool exclude rank %u tgt_idx=%d " DF_UUID ", rc=%d\n", rank, + tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } } @@ -83,6 +85,9 @@ rebuild_reint_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_reintegrate(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool reintegrate rank %u tgt_idx=%d " DF_UUID + ", rc=%d\n", + rank, tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -100,6 +105,8 @@ rebuild_extend_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_extend(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, &rank, 1); + print_message("dmg pool extend rank %u " DF_UUID ", rc=%d\n", rank, + DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -117,6 +124,8 @@ rebuild_drain_tgt(test_arg_t **args, int args_cnt, d_rank_t rank, if (!args[i]->pool.destroyed) { rc = dmg_pool_drain(args[i]->dmg_config, args[i]->pool.pool_uuid, args[i]->group, rank, tgt_idx); + print_message("dmg pool drain rank %u tgt_idx=%d " DF_UUID ", rc=%d\n", + rank, tgt_idx, DP_UUID(args[i]->pool.pool_uuid), rc); assert_success(rc); } sleep(2); @@ -162,16 +171,24 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, par_barrier(PAR_COMM_WORLD); for (i = 0; i < args_cnt; i++) - if (args[i]->rebuild_cb) + if (args[i]->rebuild_cb) { + print_message("call rebuild_cb for exclude rebuilding pool " DF_UUID + "\n", + DP_UUID(args[i]->pool.pool_uuid)); args[i]->rebuild_cb(args[i]); + } if (args[0]->myrank == 0 && !args[0]->no_rebuild) test_rebuild_wait(args, args_cnt); par_barrier(PAR_COMM_WORLD); for (i = 0; i < args_cnt; i++) { - if (args[i]->rebuild_post_cb) + if (args[i]->rebuild_post_cb) { + print_message( + "call rebuild_post_cb for exclude rebuilt pool " DF_UUID "\n", + DP_UUID(args[i]->pool.pool_uuid)); args[i]->rebuild_post_cb(args[i]); + } } return; @@ -179,6 +196,7 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, for (i = 0; i < rank_nr; i++) { int j; + const char *op_type_str = NULL; /* No concurrent drain/extend/reintegration are allowed, so * it has to reintegrate/extend one by one. @@ -187,15 +205,18 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, if (args[0]->myrank == 0) { switch (op_type) { case RB_OP_TYPE_REINT: + op_type_str = "reintegrate"; rebuild_reint_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1, kill); break; case RB_OP_TYPE_ADD: + op_type_str = "extend"; rebuild_extend_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1, args[i]->pool.pool_size); break; case RB_OP_TYPE_DRAIN: + op_type_str = "drain"; rebuild_drain_tgt(args, args_cnt, ranks[i], tgts ? tgts[i] : -1); break; @@ -208,20 +229,30 @@ rebuild_targets(test_arg_t **args, int args_cnt, d_rank_t *ranks, D_ASSERT(op_type != RB_OP_TYPE_RECLAIM); break; default: + op_type_str = "UNKNOWN"; break; } } par_barrier(PAR_COMM_WORLD); for (j = 0; j < args_cnt; j++) - if (args[j]->rebuild_cb) + if (args[j]->rebuild_cb) { + print_message("call rebuild_cb for %s rebuilding pool " DF_UUID + "\n", + op_type_str, DP_UUID(args[j]->pool.pool_uuid)); + args[j]->rebuild_cb(args[j]); + } if (args[0]->myrank == 0 && !args[0]->no_rebuild) test_rebuild_wait(args, args_cnt); for (j = 0; j < args_cnt; j++) { - if (args[j]->rebuild_post_cb) + if (args[j]->rebuild_post_cb) { + print_message("call rebuild_post_cb for %s rebuilt pool " DF_UUID + "\n", + op_type_str, DP_UUID(args[j]->pool.pool_uuid)); args[j]->rebuild_post_cb(args[j]); + } } } } @@ -965,13 +996,12 @@ reintegrate_inflight_io(void *data) SMALL_SINGLE_VALUE_SIZE, DAOS_TX_NONE, &req); sprintf(key, "d_inflight_single_large_%d", i); - insert_single(key, "a_key", 0, single_data, - LARGE_SINGLE_VALUE_SIZE, DAOS_TX_NONE, &req); - + insert_single(key, "a_key", 0, single_data, LARGE_SINGLE_VALUE_SIZE, DAOS_TX_NONE, + &req); } ioreq_fini(&req); - sleep(12); print_message("sleep 12 seconds to wait for the stable epoch update.\n"); + sleep(12); if (arg->myrank == 0) daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0, NULL); @@ -1182,82 +1212,75 @@ get_killing_rank_by_oid(test_arg_t *arg, daos_obj_id_t oid, int data_nr, *ranks_num = idx; } -/* inject fault on leader engine to stop an in-progress rebuild */ -int -fi_rebuild_stop(void *data) +static int +rebuild_stop_with_dmg_internal(const char *cfg, const uuid_t uuid, const char *grp, bool force) { - test_arg_t *arg = data; - d_rank_t leader; - int rc; + int rc; - if (arg->myrank != 0) - return 0; + rc = dmg_pool_rebuild_stop(cfg, uuid, grp, force); + print_message("dmg pool rebuild stop " DF_UUID ", force=%d, rc=%d\n", DP_UUID(uuid), force, + rc); + assert_rc_equal(rc, 0); + return 0; +} - rc = test_get_leader(arg, &leader); - if (rc) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } +/* stop an in-progress rebuild with dmg pool rebuild stop command */ +int +rebuild_stop_with_dmg(void *data) +{ + test_arg_t *arg = data; - printf("%s(): pool " DF_UUIDF ": send ADMIN_STOP to leader rank %d\n", __FUNCTION__, - DP_UUID(arg->pool.pool_uuid), leader); - test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_STOP | DAOS_FAIL_ONCE); + print_message("wait for rebuild to start for pool " DF_UUID "\n", + DP_UUID(arg->pool.pool_uuid)); + test_rebuild_wait_to_start(&arg, 1); + sleep(5); - return 0; + return rebuild_stop_with_dmg_internal(arg->dmg_config, arg->pool.pool_uuid, arg->group, + false); } -/* inject fault on leader engine to receive stop command during op:Fail_reclaim */ +/* stop an in-progress rebuild with dmg pool rebuild stop command (force stop option) */ int -fi_rebuild_stop_reclaim(void *data) +rebuild_force_stop_with_dmg(void *data) { test_arg_t *arg = data; - d_rank_t leader; - int rc; - if (arg->myrank != 0) - return 0; + print_message("wait for rebuild to start for pool " DF_UUID "\n", + DP_UUID(arg->pool.pool_uuid)); + test_rebuild_wait_to_start(&arg, 1); + sleep(5); - rc = test_get_leader(arg, &leader); - if (rc) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } + return rebuild_stop_with_dmg_internal(arg->dmg_config, arg->pool.pool_uuid, arg->group, + true); +} - printf("%s(): pool " DF_UUIDF ": send ADMIN_STOP_RECLAIM to leader rank %d\n", __FUNCTION__, - DP_UUID(arg->pool.pool_uuid), leader); - test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_STOP_RECLAIM | DAOS_FAIL_ONCE); +/* start/reesume a stopped rebuild with dmg pool rebuild start command */ +int +rebuild_start_with_dmg(void *data) +{ + test_arg_t *arg = data; + int rc; + rc = dmg_pool_rebuild_start(arg->dmg_config, arg->pool.pool_uuid, arg->group); + print_message("dmg pool rebuild start " DF_UUID ", rc=%d\n", DP_UUID(arg->pool.pool_uuid), + rc); + assert_rc_equal(rc, 0); return 0; } -/* resume after a stopped rebuild, and optionally inject a fault to start a new rebuild */ int -fi_rebuild_resume_wait(void *data) +rebuild_resume_wait(void *data) { test_arg_t *arg = data; - struct daos_rebuild_status *rst = &arg->pool.pool_info.pi_rebuild_st; - + struct daos_rebuild_status *rst = &arg->pool.pool_info.pi_rebuild_st; bool skip_restart = false; - d_rank_t leader; int rc; - if (arg->myrank != 0) - return 0; - - if (arg->rebuild_cb == fi_rebuild_resume_wait && arg->rebuild_cb_arg) + if (arg->rebuild_cb == rebuild_resume_wait && arg->rebuild_cb_arg) skip_restart = *((bool *)arg->rebuild_cb_arg); - if (arg->rebuild_post_cb == fi_rebuild_resume_wait && arg->rebuild_post_cb_arg) + if (arg->rebuild_post_cb == rebuild_resume_wait && arg->rebuild_post_cb_arg) skip_restart = *((bool *)arg->rebuild_post_cb_arg); - rc = test_get_leader(arg, &leader); - if (rc) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } - /* Verify that the stop resulted in the correct rebuild status */ print_message("check: stopped rebuild rs_errno=%d (expect %d), rs_state=%d (expect %d)\n", rst->rs_errno, -DER_OP_CANCELED, rst->rs_state, DRS_NOT_STARTED); @@ -1268,10 +1291,8 @@ fi_rebuild_resume_wait(void *data) if (skip_restart) return 0; - /* Issue start fault injection to re-run the rebuild to successful completion */ - printf("%s(): pool " DF_UUIDF ": send ADMIN_START to leader rank %d\n", __FUNCTION__, - DP_UUID(arg->pool.pool_uuid), leader); - test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_START | DAOS_FAIL_ONCE); + rc = rebuild_start_with_dmg(data); + assert_rc_equal(rc, 0); /* Verify that the start resulted in a fully completed rebuild. * NB: loop to get past the immediate remnant state from the previous stop (OP_CANCELED) diff --git a/src/tests/suite/daos_rebuild_simple.c b/src/tests/suite/daos_rebuild_simple.c index 6b95dc09047..185deda338a 100644 --- a/src/tests/suite/daos_rebuild_simple.c +++ b/src/tests/suite/daos_rebuild_simple.c @@ -108,8 +108,8 @@ rebuild_dkeys(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -170,8 +170,8 @@ rebuild_akeys(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, tgt, false); arg->rebuild_cb = NULL; @@ -222,8 +222,8 @@ rebuild_indexes(void **state) /* Rebuild rank 1 */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -283,8 +283,8 @@ rebuild_snap_update_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -349,8 +349,8 @@ rebuild_snap_punch_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -409,8 +409,8 @@ rebuild_snap_update_keys(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -517,8 +517,8 @@ rebuild_snap_punch_keys(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -606,8 +606,8 @@ rebuild_snap_punch_empty(void **state) /* stop exclude rebuild, but skip the start, then directly reintegrate */ if (arg->interactive_rebuild) { bool skip_restart = true; - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; arg->rebuild_post_cb_arg = &skip_restart; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); @@ -655,8 +655,8 @@ rebuild_snap_punch_empty(void **state) /* from a stopped (not restarted) exclude, directly reintegrate (stop+start this rebuild) */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } reintegrate_single_pool_target(arg, ranks_to_kill[0], tgt); rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX); @@ -705,8 +705,8 @@ rebuild_multiple(void **state) } if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -763,8 +763,8 @@ rebuild_large_rec(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -803,8 +803,8 @@ rebuild_objects(void **state) rebuild_io(arg, oids, OBJ_NR); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); arg->rebuild_cb = NULL; @@ -844,6 +844,12 @@ rebuild_sx_object_internal(void **state, daos_oclass_id_t oclass, if (!test_runable(arg, 4)) return; + if (arg->interactive_rebuild && !wait_rebuild) { + print_message("SKIP due to interactive_rebuild enabled, but not tested here\n"); + skip(); + return; + } + oid = daos_test_oid_gen(arg->coh, oclass, 0, 0, arg->myrank); ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); print_message("insert 100 dkeys\n"); @@ -874,30 +880,35 @@ rebuild_sx_object_internal(void **state, daos_oclass_id_t oclass, get_killing_rank_by_oid(arg, oid, 1, 0, &rank, &rank_nr); /** exclude the target of this obj's replicas */ - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool exclude rank %u " DF_UUID "\n", rank, DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_exclude(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until exclude rebuild done */ if (wait_rebuild) { test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); } - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool reintegrate rank %u " DF_UUID "\n", rank, + DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_reintegrate(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until reintegration rebuild is done */ if (wait_rebuild) { test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); } print_message("lookup 100 dkeys\n"); @@ -978,27 +989,33 @@ rebuild_large_object(void **state) ioreq_fini(&req); } - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool exclude rank %u " DF_UUID "\n", rank, DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_exclude(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) + rebuild_stop_with_dmg(arg); + /* wait until exclude rebuild done */ test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); - if (arg->interactive_rebuild) - fi_rebuild_stop(arg); + print_message("dmg pool reintegrate rank %u " DF_UUID "\n", rank, + DP_UUID(arg->pool.pool_uuid)); rc = dmg_pool_reintegrate(arg->dmg_config, arg->pool.pool_uuid, arg->group, rank, -1); assert_success(rc); + if (arg->interactive_rebuild) { + rebuild_stop_with_dmg(arg); + } + /* wait until reintegration rebuild is done */ test_rebuild_wait(&arg, 1); if (arg->interactive_rebuild) - fi_rebuild_resume_wait(arg); + rebuild_resume_wait(arg); print_message("success\n"); } @@ -1094,8 +1111,8 @@ rebuild_large_snap(void **state) /* stop and resume the exclude rebuild */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], tgt, false); ioreq_fini(&req); @@ -1142,8 +1159,8 @@ rebuild_full_shards(void **state) /* rebuild and reintegration to use full shards */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } /* stop and resume the first exclude rebuild (but not the second one) */ rebuild_single_pool_target(arg, 0, -1, false); @@ -1153,8 +1170,8 @@ rebuild_full_shards(void **state) /* stop and resume the first reintegrate rebuild (but not the second one) */ if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } reintegrate_single_pool_target(arg, 0, -1); arg->rebuild_cb = NULL; @@ -1196,8 +1213,8 @@ rebuild_punch_recs(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, ranks_to_kill[0], -1, false); arg->rebuild_cb = NULL; @@ -1253,8 +1270,8 @@ rebuild_multiple_group(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1264,10 +1281,6 @@ rebuild_multiple_group(void **state) if (rc != 0) assert_rc_equal(rc, -DER_NOSYS); - if (arg->interactive_rebuild) { - arg->rebuild_cb = NULL; - arg->rebuild_post_cb = NULL; - } reintegrate_with_inflight_io(arg, &oid, kill_rank, -1); rc = daos_obj_verify(arg->coh, oid, DAOS_EPOCH_MAX); @@ -1311,8 +1324,8 @@ rebuild_with_large_offset(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1363,8 +1376,8 @@ rebuild_with_large_key(void **state) ioreq_fini(&req); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rebuild_single_pool_target(arg, kill_rank, -1, false); arg->rebuild_cb = NULL; @@ -1432,8 +1445,8 @@ rebuild_with_dfs_open_create_punch(void **state) dfs_obj2id(dir, &oid); if (arg->interactive_rebuild) { - arg->rebuild_cb = fi_rebuild_stop; - arg->rebuild_post_cb = fi_rebuild_resume_wait; + arg->rebuild_cb = rebuild_stop_with_dmg; + arg->rebuild_post_cb = rebuild_resume_wait; } rank = get_rank_by_oid_shard(arg, oid, 0); rebuild_single_pool_rank(arg, rank, false); @@ -1487,6 +1500,40 @@ rebuild_wait_reset_fail_cb(void *data) return 0; } +static int +rebuild_wait_error_reset_fail_cb(void *data) +{ + test_arg_t *arg = data; + int rc; + + print_message("wait until rebuild errors (and starts Fail_reclaim)\n"); + test_rebuild_wait_to_error(&arg, 1); + print_message("check rebuild errored, rs_errno=%d (expecting -DER_IO=%d)\n", + arg->pool.pool_info.pi_rebuild_st.rs_errno, -DER_IO); + assert_int_equal(arg->pool.pool_info.pi_rebuild_st.rs_errno, -DER_IO); + print_message("rebuild error code check passed\n"); + + print_message("clearing fault injection on all engines\n"); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0, NULL); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 0, 0, NULL); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_NUM, 0, 0, NULL); + + /* Give time for transition from op:Rebuild into op:Fail_reclaim */ + sleep(2); + + print_message( + "send rebuild stop --force request during first/only Fail_reclaim operation\n"); + rc = rebuild_force_stop_with_dmg(data); + if (rc != 0) + print_message("rebuild_force_stop_with_dmg failed, rc=%d\n", rc); + + print_message("wait for rebuild to be stopped\n"); + test_rebuild_wait(&arg, 1); + /* Verifying rs_state/rs_errno will happen in post_cb rebuild_resume_wait() */ + + return rc; +} + static void rebuild_many_objects_with_failure(void **state) { @@ -1516,46 +1563,25 @@ rebuild_many_objects_with_failure(void **state) /* Inject faults on engines. Special handling for interactive_rebuild case */ if (arg->myrank == 0) { - if (!arg->interactive_rebuild) { - /* All engines DAOS_REBUILD_OBJ_FAIL */ - print_message("inject fault DAOS_REBUILD_OBJ_FAIL on all engines\n"); - daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, - DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); - daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 50, 0, NULL); - } else { - d_rank_t leader; - - /* interactive_rebiuld: DAOS_REBUILD_OBJ fail on all engines except PS - * leader; DAOS_REBUILD_ADMIN_STOP_RECLAIM on the PS leader engine - */ - - rc = test_get_leader(arg, &leader); - if (rc != 0) { - print_message("get pool " DF_UUIDF " leader failed: %d\n", - DP_UUID(arg->pool.pool_uuid), rc); - assert_rc_equal(rc, 0); - } - - for (i = 0; i < arg->srv_nnodes; i++) { - if (i == leader) - fi_rebuild_stop_reclaim(arg); - else { - print_message("inject DAOS_REBUILD_OBJ_FAIL on non-leader " - "engine %u\n", - i); - daos_debug_set_params( - arg->group, i, DMG_KEY_FAIL_LOC, - DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); - daos_debug_set_params(arg->group, i, DMG_KEY_FAIL_VALUE, 50, - 0, NULL); - } - } - } + print_message("inject fault DAOS_REBUILD_OBJ_FAIL on all engines\n"); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, + DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL); + daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 50, 0, NULL); } - arg->rebuild_cb = rebuild_wait_reset_fail_cb; - if (arg->interactive_rebuild) - arg->rebuild_post_cb = fi_rebuild_resume_wait; + /* For interactive rebuild, we need: + * 1. trigger rebuild (which will fail), query pool reubild state until op:Rebuild fails + * and op:Fail_reclaim begins. See test_rebuild_wait_to_error(). + * 2. Then, while rebuild is in op:Fail_reclaim, issue dmg system stop to test that you + * can't stop during Fail_reclaim (though the command will take effect by not retrying + * rebuild). + */ + if (arg->interactive_rebuild) { + arg->rebuild_cb = rebuild_wait_error_reset_fail_cb; + arg->rebuild_post_cb = rebuild_resume_wait; + } else { + arg->rebuild_cb = rebuild_wait_reset_fail_cb; + } rebuild_single_pool_target(arg, 3, -1, false); for (i = 0; i < 8000; i++) { diff --git a/src/tests/suite/daos_test.h b/src/tests/suite/daos_test.h index 97a56bdec7d..b6f15f06fbb 100644 --- a/src/tests/suite/daos_test.h +++ b/src/tests/suite/daos_test.h @@ -397,6 +397,10 @@ int test_pool_get_info(test_arg_t *arg, daos_pool_info_t *pinfo, d_rank_list_t * int test_get_leader(test_arg_t *arg, d_rank_t *rank); bool test_rebuild_query(test_arg_t **args, int args_cnt); void test_rebuild_wait(test_arg_t **args, int args_cnt); +void +test_rebuild_wait_to_start(test_arg_t **args, int args_cnt); +void + test_rebuild_wait_to_error(test_arg_t **args, int args_cnt); int daos_pool_set_prop(const uuid_t pool_uuid, const char *name, const char *value); @@ -456,24 +460,34 @@ void rebuild_add_back_tgts(test_arg_t *arg, d_rank_t failed_rank, int rebuild_pool_disconnect_internal(void *data); int rebuild_pool_connect_internal(void *data); - -int rebuild_sub_setup(void **state); -int rebuild_sub_rf1_setup(void **state); -int rebuild_sub_rf0_setup(void **state); -int rebuild_sub_teardown(void **state); -int rebuild_small_sub_setup(void **state); -int rebuild_small_sub_rf1_setup(void **state); -int rebuild_small_sub_rf0_setup(void **state); -int rebuild_sub_3nodes_rf0_setup(void **state); -int rebuild_sub_6nodes_rf1_setup(void **state); -int rebuild_sub_setup_common(void **state, daos_size_t pool_size, int node_nr, uint32_t rf); - int -fi_rebuild_stop(void *data); +rebuild_sub_setup(void **state); +int +rebuild_sub_rf1_setup(void **state); +int +rebuild_sub_rf0_setup(void **state); +int +rebuild_sub_teardown(void **state); +int +rebuild_small_sub_setup(void **state); +int +rebuild_small_sub_rf1_setup(void **state); +int +rebuild_small_sub_rf0_setup(void **state); +int +rebuild_sub_3nodes_rf0_setup(void **state); +int +rebuild_sub_6nodes_rf1_setup(void **state); +int +rebuild_sub_setup_common(void **state, daos_size_t pool_size, int node_nr, uint32_t rf); +int +rebuild_stop_with_dmg(void *data); +int +rebuild_force_stop_with_dmg(void *data); int -fi_rebuild_stop_reclaim(void *data); +rebuild_start_with_dmg(void *data); int - fi_rebuild_resume_wait(void *data); + rebuild_resume_wait(void *data); int get_server_config(char *host, char *server_config_file); int get_log_file(char *host, char *server_config_file, diff --git a/src/tests/suite/daos_test_common.c b/src/tests/suite/daos_test_common.c index 2f6b3fd3691..21bd5a7fb7d 100644 --- a/src/tests/suite/daos_test_common.c +++ b/src/tests/suite/daos_test_common.c @@ -791,7 +791,61 @@ test_pool_get_info(test_arg_t *arg, daos_pool_info_t *pinfo, d_rank_list_t **eng } static bool -rebuild_pool_wait(test_arg_t *arg) +rebuild_pool_started(test_arg_t *arg) +{ + daos_pool_info_t pinfo = {0}; + struct daos_rebuild_status *rst; + int rc; + + pinfo.pi_bits = DPI_REBUILD_STATUS; + rc = test_pool_get_info(arg, &pinfo, NULL /* engine_ranks */); + rst = &pinfo.pi_rebuild_st; + + if (rc != 0) { + print_message("pool query for rebuild status failed, rc=%d, pool " DF_UUIDF "\n", + rc, DP_UUID(arg->pool.pool_uuid)); + return false; + } else { + bool started = (rst->rs_state == DRS_IN_PROGRESS); + print_message("rebuild for pool " DF_UUIDF "has %sstarted\n", + DP_UUID(arg->pool.pool_uuid), started ? "" : "not yet "); + return started; + } +} + +static bool +rebuild_pool_erroring(test_arg_t *arg) +{ + daos_pool_info_t pinfo = {0}; + struct daos_rebuild_status *rst; + int rc; + + pinfo.pi_bits = DPI_REBUILD_STATUS; + rc = test_pool_get_info(arg, &pinfo, NULL /* engine_ranks */); + rst = &pinfo.pi_rebuild_st; + + if (rc != 0) { + print_message("pool query for rebuild status failed, rc=%d, pool " DF_UUIDF "\n", + rc, DP_UUID(arg->pool.pool_uuid)); + return false; + } else { + bool started = (rst->rs_state == DRS_IN_PROGRESS); + bool erroring = started && (rst->rs_errno != 0); + + print_message("rebuild for pool " DF_UUIDF "has %sstarted, rs_errno=%d\n", + DP_UUID(arg->pool.pool_uuid), started ? "" : "not yet ", + rst->rs_errno); + + /* save final pool query info to be able to inspect rebuild status */ + if (erroring) + memcpy(&arg->pool.pool_info, &pinfo, sizeof(pinfo)); + + return erroring; + } +} + +static bool +rebuild_pool_done(test_arg_t *arg) { daos_pool_info_t pinfo = {0}; struct daos_rebuild_status *rst; @@ -862,6 +916,56 @@ test_get_last_svr_rank(test_arg_t *arg) return arg->srv_nnodes - disable_nodes - 1; } +bool +test_rebuild_started(test_arg_t **args, int args_cnt) +{ + bool all_started = true; + int i; + + for (i = 0; i < args_cnt; i++) { + bool started = true; + + if (!args[i]->pool.destroyed) + started = rebuild_pool_started(args[i]); + + if (!started) + all_started = false; + } + return all_started; +} + +void +test_rebuild_wait_to_start(test_arg_t **args, int args_cnt) +{ + while (!test_rebuild_started(args, args_cnt)) + sleep(2); +} + +bool +test_rebuild_erroring(test_arg_t **args, int args_cnt) +{ + bool all_erroring = true; + int i; + + for (i = 0; i < args_cnt; i++) { + bool erroring = true; + + if (!args[i]->pool.destroyed) + erroring = rebuild_pool_erroring(args[i]); + + if (!erroring) + all_erroring = false; + } + return all_erroring; +} + +void +test_rebuild_wait_to_error(test_arg_t **args, int args_cnt) +{ + while (!test_rebuild_erroring(args, args_cnt)) + sleep(1); +} + bool test_rebuild_query(test_arg_t **args, int args_cnt) { @@ -872,7 +976,7 @@ test_rebuild_query(test_arg_t **args, int args_cnt) bool done = true; if (!args[i]->pool.destroyed) - done = rebuild_pool_wait(args[i]); + done = rebuild_pool_done(args[i]); if (!done) all_done = false;