DAOS-17281 rebuild: handle admin stop during op:Fail_reclaim

src/include/daos/common.h

@@ -822,6 +822,7 @@ enum {
  */
 #define DAOS_FAIL_PARITY_EPOCH_DIFF	(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x29)
 #define DAOS_FAIL_SHARD_NONEXIST	(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x2a)
+#define DAOS_REBUILD_ADMIN_STOP_RECLAIM   (DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x2b)
 
 #define DAOS_DTX_COMMIT_SYNC		(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x30)
 #define DAOS_DTX_LEADER_ERROR		(DAOS_FAIL_UNIT_TEST_GROUP_LOC | 0x31)

src/include/daos_srv/rebuild.h

@@ -86,7 +86,8 @@ typedef enum {
 int
      ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, daos_epoch_t stable_eph,
 			 uint32_t layout_version, struct pool_target_id_list *tgts,
-			 daos_rebuild_opc_t rebuild_op, bool stop_admin, uint64_t delay_sec);
+			 daos_rebuild_opc_t rebuild_op, daos_rebuild_opc_t retry_rebuild_op,
+			 uint32_t retry_map_ver, bool stop_admin, uint64_t delay_sec);
 void ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank);
 int ds_rebuild_query(uuid_t pool_uuid,
 		     struct daos_rebuild_status *status);

src/pool/srv_pool.c

@@ -6409,6 +6409,7 @@ pool_check_upgrade_object_layout(struct rdb_tx *tx, struct pool_svc *svc,
 	if (current_layout_ver < DS_POOL_OBJ_VERSION) {
 		rc = ds_rebuild_schedule(svc->ps_pool, svc->ps_pool->sp_map_version, upgrade_eph,
 					 DS_POOL_OBJ_VERSION, NULL, RB_OP_UPGRADE,
+					 RB_OP_NONE /* retry_rebuild_op */, 0 /* retry_map_ver */,
 					 false /* stop_admin */, 0);
 		if (rc == 0)
 			*scheduled_layout_upgrade = true;
@@ -7756,7 +7757,8 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank,
 
 	if (tgt_map_ver != 0) {
 		rc = ds_rebuild_schedule(svc->ps_pool, tgt_map_ver, rebuild_eph, 0, &target_list,
-					 RB_OP_REBUILD, false /* stop_admin */, delay);
+					 RB_OP_REBUILD, RB_OP_NONE /* retry_rebuild_op */,
+					 0 /* retry_map_ver */, false /* stop_admin */, delay);
 		if (rc != 0) {
 			D_ERROR("rebuild fails rc: "DF_RC"\n", DP_RC(rc));
 			D_GOTO(out, rc);

src/rebuild/rebuild_internal.h

@@ -245,8 +245,13 @@ struct rebuild_task {
 	 */
 	uint32_t			dst_reclaim_ver;
 
+	/* For failed tasks that may be retried after RB_OP_FAIL_RECLAIM, original map ver and opc
+	 */
+	uint32_t                        dst_retry_map_ver;
+	daos_rebuild_opc_t              dst_retry_rebuild_op;
+
 	/* For dst_rebuild_op == RB_OP_FAIL_RECLAIM: If true, rebuild was stopped by admin.
-	 * Then, on fail_reclaim finish, the pool rebuild state will be set to completed.
+	 * Then, on fail_reclaim finish, the pool rebuild state will be set to idle (NOT_STARTED).
 	 */
 	bool                            dst_stop_admin;
 };

src/tests/suite/daos_rebuild_common.c

@@ -1207,6 +1207,31 @@ fi_rebuild_stop(void *data)
 	return 0;
 }
 
+/* inject fault on leader engine to receive stop command during op:Fail_reclaim */
+int
+fi_rebuild_stop_reclaim(void *data)
+{
+	test_arg_t *arg = data;
+	d_rank_t    leader;
+	int         rc;
+
+	if (arg->myrank != 0)
+		return 0;
+
+	rc = test_get_leader(arg, &leader);
+	if (rc) {
+		print_message("get pool " DF_UUIDF " leader failed: %d\n",
+			      DP_UUID(arg->pool.pool_uuid), rc);
+		assert_rc_equal(rc, 0);
+	}
+
+	printf("%s(): pool " DF_UUIDF ": send ADMIN_STOP_RECLAIM to leader rank %d\n", __FUNCTION__,
+	       DP_UUID(arg->pool.pool_uuid), leader);
+	test_set_engine_fail_loc(arg, leader, DAOS_REBUILD_ADMIN_STOP_RECLAIM | DAOS_FAIL_ONCE);
+
+	return 0;
+}
+
 /* resume after a stopped rebuild, and optionally inject a fault to start a new rebuild */
 int
 fi_rebuild_resume_wait(void *data)

src/tests/suite/daos_rebuild_simple.c

@@ -1475,9 +1475,11 @@ rebuild_wait_reset_fail_cb(void *data)
 {
 	test_arg_t	*arg = data;
 
-	print_message("wait 300 seconds for rebuild/reclaim/retry....");
+	print_message("wait 60 seconds for rebuild/reclaim%s\n",
+		      arg->interactive_rebuild ? "" : "/retry");
 	sleep(60);
 
+	print_message("clearing fault injection on all engines\n");
 	daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, 0, 0, NULL);
 	daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 0, 0, NULL);
 	daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_NUM, 0, 0, NULL);
@@ -1496,11 +1498,6 @@ rebuild_many_objects_with_failure(void **state)
 	if (!test_runable(arg, 6))
 		return;
 
-	if (arg->interactive_rebuild) {
-		print_message("SKIP due to interactive_rebuild enabled, but not tested here\n");
-		skip();
-	}
-
 	D_ALLOC_ARRAY(oids, 8000);
 	for (i = 0; i < 8000; i++) {
 		char buffer[256];
@@ -1517,15 +1514,48 @@ rebuild_many_objects_with_failure(void **state)
 		ioreq_fini(&req);
 	}
 
+	/* Inject faults on engines. Special handling for interactive_rebuild case */
 	if (arg->myrank == 0) {
-		daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC,
-				      DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL);
-		daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 50,
-				      0, NULL);
+		if (!arg->interactive_rebuild) {
+			/* All engines DAOS_REBUILD_OBJ_FAIL */
+			print_message("inject fault DAOS_REBUILD_OBJ_FAIL on all engines\n");
+			daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC,
+					      DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL);
+			daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 50, 0, NULL);
+		} else {
+			d_rank_t leader;
+
+			/* interactive_rebiuld: DAOS_REBUILD_OBJ fail on all engines except PS
+			 * leader; DAOS_REBUILD_ADMIN_STOP_RECLAIM on the PS leader engine
+			 */
+
+			rc = test_get_leader(arg, &leader);
+			if (rc != 0) {
+				print_message("get pool " DF_UUIDF " leader failed: %d\n",
+					      DP_UUID(arg->pool.pool_uuid), rc);
+				assert_rc_equal(rc, 0);
+			}
+
+			for (i = 0; i < arg->srv_nnodes; i++) {
+				if (i == leader)
+					fi_rebuild_stop_reclaim(arg);
+				else {
+					print_message("inject DAOS_REBUILD_OBJ_FAIL on non-leader "
+						      "engine %u\n",
+						      i);
+					daos_debug_set_params(
+					    arg->group, i, DMG_KEY_FAIL_LOC,
+					    DAOS_REBUILD_OBJ_FAIL | DAOS_FAIL_ALWAYS, 0, NULL);
+					daos_debug_set_params(arg->group, i, DMG_KEY_FAIL_VALUE, 50,
+							      0, NULL);
+				}
+			}
+		}
 	}
 
 	arg->rebuild_cb = rebuild_wait_reset_fail_cb;
-
+	if (arg->interactive_rebuild)
+		arg->rebuild_post_cb = fi_rebuild_resume_wait;
 	rebuild_single_pool_target(arg, 3, -1, false);
 
 	for (i = 0; i < 8000; i++) {

src/tests/suite/daos_test.h

@@ -470,6 +470,8 @@ int rebuild_sub_setup_common(void **state, daos_size_t pool_size, int node_nr, u
 
 int
 fi_rebuild_stop(void *data);
+int
+fi_rebuild_stop_reclaim(void *data);
 int
      fi_rebuild_resume_wait(void *data);