DAOS-17357 rebuild: use dmg for stop/start, force-stop fix

src/common/tests_dmg_helpers.c

@@ -1206,6 +1206,80 @@ dmg_pool_list(const char *dmg_config_file, const char *group,
 	return rc;
 }
 
+int
+dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force)
+{
+	char                uuid_str[DAOS_UUID_STR_SIZE];
+	int                 argcount = 0;
+	char              **args     = NULL;
+	struct json_object *dmg_out  = NULL;
+	int                 rc       = 0;
+
+	uuid_unparse_lower(uuid, uuid_str);
+	args = cmd_push_arg(args, &argcount, "%s ", uuid_str);
+	if (args == NULL)
+		D_GOTO(out, rc = -DER_NOMEM);
+
+	if (grp != NULL) {
+		args = cmd_push_arg(args, &argcount, "--sys=%s ", grp);
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	if (force) {
+		args = cmd_push_arg(args, &argcount, "--force");
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	rc = daos_dmg_json_pipe("pool rebuild stop", dmg_config_file, args, argcount, &dmg_out);
+	if (rc != 0) {
+		D_ERROR("dmg pool rebuild stop failed\n");
+		goto out_json;
+	}
+
+out_json:
+	if (dmg_out != NULL)
+		json_object_put(dmg_out);
+	cmd_free_args(args, argcount);
+out:
+	return rc;
+}
+
+int
+dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp)
+{
+	char                uuid_str[DAOS_UUID_STR_SIZE];
+	int                 argcount = 0;
+	char              **args     = NULL;
+	struct json_object *dmg_out  = NULL;
+	int                 rc       = 0;
+
+	uuid_unparse_lower(uuid, uuid_str);
+	args = cmd_push_arg(args, &argcount, "%s ", uuid_str);
+	if (args == NULL)
+		D_GOTO(out, rc = -DER_NOMEM);
+
+	if (grp != NULL) {
+		args = cmd_push_arg(args, &argcount, "--sys=%s ", grp);
+		if (args == NULL)
+			D_GOTO(out, rc = -DER_NOMEM);
+	}
+
+	rc = daos_dmg_json_pipe("pool rebuild start", dmg_config_file, args, argcount, &dmg_out);
+	if (rc != 0) {
+		D_ERROR("dmg pool rebuild start failed\n");
+		goto out_json;
+	}
+
+out_json:
+	if (dmg_out != NULL)
+		json_object_put(dmg_out);
+	cmd_free_args(args, argcount);
+out:
+	return rc;
+}
+
 static int
 parse_device_info(struct json_object *smd_dev, device_list *devices,
 		  char *host, int dev_length, int *disks)

src/include/daos/common.h

@@ -789,9 +789,6 @@ enum {
 #define DAOS_RDB_SKIP_APPENDENTRIES_FAIL (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1a)
 #define DAOS_FORCE_REFRESH_POOL_MAP	  (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1b)
 
-#define DAOS_REBUILD_ADMIN_STOP           (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1c)
-#define DAOS_REBUILD_ADMIN_START          (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1d)
-
 #define DAOS_FORCE_CAPA_FETCH		(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1e)
 #define DAOS_FORCE_PROP_VERIFY		(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x1f)
 

src/include/daos/tests_lib.h

@@ -484,6 +484,25 @@ dmg_pool_set_prop(const char *dmg_config_file,
 int dmg_pool_get_prop(const char *dmg_config_file, const char *label, const uuid_t uuid,
 		      const char *name, char **value);
 
+/**
+ * Interactively stop a pool's currently-running rebuild.
+ * \param dmg_config_file [IN] DMG config file.
+ * \param uuid            [IN] UUID of the pool.
+ * \param grp             [IN] Process set name of the DAOS servers managing the pool.
+ * \param force           [IN] forcibly stop a rebuild that is failing.
+ */
+int
+dmg_pool_rebuild_stop(const char *dmg_config_file, const uuid_t uuid, const char *grp, bool force);
+
+/**
+ * Interactively start/resume a pool's rebuilding.
+ * \param dmg_config_file [IN] DMG config file.
+ * \param uuid            [IN] UUID of the pool.
+ * \param grp             [IN] Process set name of the DAOS servers managing the pool.
+ */
+int
+    dmg_pool_rebuild_start(const char *dmg_config_file, const uuid_t uuid, const char *grp);
+
 /**
  * List all disks in the specified DAOS system.
  *

src/include/daos_srv/rebuild.h

@@ -87,7 +87,7 @@ int
      ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t stable_eph,
 			 uint32_t layout_version, struct pool_target_id_list *tgts,
 			 daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op,
-			 uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec);
+			 uint32_t retry_map_ver, bool stop_admin, void *cur_taskp, uint64_t delay_sec);
 void ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank);
 int ds_rebuild_query(uuid_t pool_uuid,
 		     struct daos_rebuild_status *status);

src/pool/srv_pool.c

@@ -6410,7 +6410,7 @@ pool_check_upgrade_object_layout(struct rdb_tx *tx, struct pool_svc *svc,
 		rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, upgrade_eph,
 					 DS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE,
 					 RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */,
-					 false /* stop_admin */, 0);
+					 false /* stop_admin */, NULL /* cur_taskp */, 0);
 		if (rc == 0)
 			*scheduled_layout_upgrade = true;
 	}
@@ -7758,7 +7758,8 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank,
 	if (tgt_map_ver != 0) {
 		rc = ds_rebuild_schedule(svc->ps_pool, tgt_map_ver, rebuild_eph, 0, &target_list,
 					 RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */,
-					 0 /* retry_map_ver */, false /* stop_admin */, delay);
+					 0 /* retry_map_ver */, false /* stop_admin */,
+					 NULL /* cur_taskp */, delay);
 		if (rc != 0) {
 			D_ERROR("rebuild fails rc: "DF_RC"\n", DP_RC(rc));
 			D_GOTO(out, rc);

src/rebuild/rebuild_internal.h

@@ -168,6 +168,11 @@ struct rebuild_global_pool_tracker {
 	uint32_t	rgt_opc;
 	unsigned int                    rgt_abort : 1, /* abort: kill rebuild */
 	    rgt_init_scan : 1, rgt_stop_admin : 1;     /* stop: admin has asked to kill rebuild */
+
+	uint32_t rgt_num_op_rb;            /* count of op:Rebuild attempts */
+	uint32_t rgt_num_op_freclaim;      /* count of op:Fail_reclaim attempts */
+	uint32_t rgt_num_op_rb_fail;       /* count of op:Rebuild failures */
+	uint32_t rgt_num_op_freclaim_fail; /* count of failed op:Fail_reclaim (not good) */
 };
 
 /* Structure on raft replica nodes to serve completed rebuild status querying */
@@ -254,6 +259,14 @@ struct rebuild_task {
 	 * Then, on fail_reclaim finish, the pool rebuild state will be set to idle (NOT_STARTED).
 	 */
 	bool                            dst_stop_admin;
+
+	/* Track how many tries for certain daos_rebuild_opc_t */
+	uint32_t                        dst_num_op_rb; /* count of tries to run rebuild */
+	uint32_t                        dst_num_op_reclaim;
+	uint32_t                        dst_num_op_freclaim;
+	uint32_t                        dst_num_op_upgrade;
+	uint32_t                        dst_num_op_rb_fail; /* count of rebuild failures */
+	uint32_t dst_num_op_freclaim_fail;                  /* count of Fail_recliam failures */
 };
 
 /* Per pool structure in TLS to check pool rebuild status