DAOS-17788 test: Fix recovery/pool_list_consolidation.py test_lost_al…

src/tests/ftest/recovery/pool_list_consolidation.py

@@ -11,7 +11,7 @@
 from ClusterShell.NodeSet import NodeSet
 from general_utils import check_file_exists, report_errors
 from recovery_utils import wait_for_check_complete
-from run_utils import run_remote
+from run_utils import command_as_user, run_remote
 
 
 class PoolListConsolidationTest(TestWithServers):
@@ -360,10 +360,9 @@ def test_lost_all_rdb(self):
         1. Create a pool.
         2. Stop servers.
         3. Remove <scm_mount>/<pool_uuid>/rdb-pool from all ranks.
-        4. Start servers.
-        5. Run DAOS checker under kinds of mode.
-        6. Check that the pool does not appear with dmg pool list.
-        7. Verify that the pool directory was removed from the mount point.
+        4. Run DAOS checker under kinds of mode.
+        5. Check that the pool does not appear with dmg pool list.
+        6. Verify that the pool directory was removed from the mount point.
 
         Jira ID: DAOS-12067
 
@@ -373,36 +372,33 @@ def test_lost_all_rdb(self):
         :avocado: tags=PoolListConsolidationTest,test_lost_all_rdb
         """
         self.log_step("Create a pool.")
-        pool = self.get_pool()
+        pool = self.get_pool(connect=False)
 
         self.log_step("Stop servers.")
         dmg_command = self.get_dmg_command()
         dmg_command.system_stop()
 
         self.log_step("Remove <scm_mount>/<pool_uuid>/rdb-pool from all ranks.")
-        rdb_pool_path = f"{self.server_managers[0].get_vos_path(pool)}/rdb-pool"
-        rdb_pool_out = check_file_exists(
-            hosts=self.hostlist_servers, filename=rdb_pool_path, sudo=True)
-        if not rdb_pool_out[0]:
+        scm_mounts = self.server_managers[0].manager.job.yaml.get_engine_values(
+            "scm_mount")
+        rdb_pool_paths = [f"{scm}/{pool.uuid.lower()}/rdb-pool" for scm in scm_mounts]
+        self.log.info("rdb_pool_paths: %s", rdb_pool_paths)
+        rdb_pool_exists = [check_file_exists(
+            self.hostlist_servers, path, sudo=True)[0] for path in rdb_pool_paths]
+        if not all(rdb_pool_exists):
             msg = ("MD-on-SSD cluster. Contents under mount point are removed by control plane "
                    "after system stop.")
             self.log.info(msg)
             dmg_command.system_start()
             # return results in PASS.
             return
-        command = f"sudo rm {rdb_pool_path}"
-        remove_result = run_remote(log=self.log, hosts=self.hostlist_servers, command=command)
-        if not remove_result.passed:
-            self.fail(f"Failed to remove {rdb_pool_path} from {self.hostlist_servers}")
-        success_nodes = remove_result.passed_hosts
-        if self.hostlist_servers != success_nodes:
-            msg = (f"Failed to remove rdb-pool! All = {self.hostlist_servers}, "
-                   f"Success = {success_nodes}")
-            self.fail(msg)
-
-        # 4. Start servers.
-        self.log_step("Start servers.")
-        dmg_command.system_start()
+        for rdb_pool_path in rdb_pool_paths:
+            command = command_as_user(command=f"rm {rdb_pool_path}", user="root")
+            remove_result = run_remote(
+                log=self.log, hosts=self.hostlist_servers, command=command)
+            if not remove_result.passed:
+                self.fail(
+                    f"Failed to remove {rdb_pool_path} from {remove_result.failed_hosts}")
 
         self.log_step("Run DAOS checker under kinds of mode.")
         errors = []