From 4e2656115c3ebf52500841b50f332b1822cefca0 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Thu, 18 Sep 2025 07:24:05 +0000 Subject: [PATCH 1/6] DAOS-17788 test: Fix recovery/pool_list_consolidation.py test_lost_all_rdb The test should remove rdb-pool from all ranks from all nodes, but it was only removing it from one of the two ranks in a node even though the test uses dual engine. Fix it to remove from both ranks. Remove unnecessary system start step. Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_lost_all_rdb Signed-off-by: Makito Kano --- .../ftest/recovery/pool_list_consolidation.py | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/tests/ftest/recovery/pool_list_consolidation.py b/src/tests/ftest/recovery/pool_list_consolidation.py index 0633be08486..49aa87922a8 100644 --- a/src/tests/ftest/recovery/pool_list_consolidation.py +++ b/src/tests/ftest/recovery/pool_list_consolidation.py @@ -360,10 +360,9 @@ def test_lost_all_rdb(self): 1. Create a pool. 2. Stop servers. 3. Remove //rdb-pool from all ranks. - 4. Start servers. - 5. Run DAOS checker under kinds of mode. - 6. Check that the pool does not appear with dmg pool list. - 7. Verify that the pool directory was removed from the mount point. + 4. Run DAOS checker under kinds of mode. + 5. Check that the pool does not appear with dmg pool list. + 6. Verify that the pool directory was removed from the mount point. Jira ID: DAOS-12067 @@ -373,36 +372,42 @@ def test_lost_all_rdb(self): :avocado: tags=PoolListConsolidationTest,test_lost_all_rdb """ self.log_step("Create a pool.") - pool = self.get_pool() + pool = self.get_pool(connect=False) self.log_step("Stop servers.") dmg_command = self.get_dmg_command() dmg_command.system_stop() self.log_step("Remove //rdb-pool from all ranks.") - rdb_pool_path = f"{self.server_managers[0].get_vos_path(pool)}/rdb-pool" - rdb_pool_out = check_file_exists( - hosts=self.hostlist_servers, filename=rdb_pool_path, sudo=True) - if not rdb_pool_out[0]: + engine_params = self.server_managers[0].manager.job.yaml.engine_params + scm_mount_0 = engine_params[0].get_value("scm_mount") + scm_mount_1 = engine_params[1].get_value("scm_mount") + rdb_pool_path_0 = f"{scm_mount_0}/{pool.uuid.lower()}/rdb-pool" + rdb_pool_path_1 = f"{scm_mount_1}/{pool.uuid.lower()}/rdb-pool" + self.log.info("rdb_pool_path_0 = %s", rdb_pool_path_0) + self.log.info("rdb_pool_path_1 = %s", rdb_pool_path_1) + rdb_pool_out_0 = check_file_exists( + hosts=self.hostlist_servers, filename=rdb_pool_path_0, sudo=True) + rdb_pool_out_1 = check_file_exists( + hosts=self.hostlist_servers, filename=rdb_pool_path_1, sudo=True) + if not rdb_pool_out_0[0] or not rdb_pool_out_1[0]: msg = ("MD-on-SSD cluster. Contents under mount point are removed by control plane " "after system stop.") self.log.info(msg) dmg_command.system_start() # return results in PASS. return - command = f"sudo rm {rdb_pool_path}" - remove_result = run_remote(log=self.log, hosts=self.hostlist_servers, command=command) - if not remove_result.passed: - self.fail(f"Failed to remove {rdb_pool_path} from {self.hostlist_servers}") - success_nodes = remove_result.passed_hosts - if self.hostlist_servers != success_nodes: - msg = (f"Failed to remove rdb-pool! All = {self.hostlist_servers}, " - f"Success = {success_nodes}") - self.fail(msg) - - # 4. Start servers. - self.log_step("Start servers.") - dmg_command.system_start() + rdb_pool_path_list = [rdb_pool_path_0, rdb_pool_path_1] + for rdb_pool_path in rdb_pool_path_list: + command = f"sudo rm {rdb_pool_path}" + remove_result = run_remote(log=self.log, hosts=self.hostlist_servers, command=command) + if not remove_result.passed: + self.fail(f"Failed to remove {rdb_pool_path} from {self.hostlist_servers}") + success_nodes = remove_result.passed_hosts + if self.hostlist_servers != success_nodes: + msg = (f"Failed to remove rdb-pool! All = {self.hostlist_servers}, " + f"Success = {success_nodes}") + self.fail(msg) self.log_step("Run DAOS checker under kinds of mode.") errors = [] From 06b6bae5fac4ab2c618ddfdaba24ceabeb7d13f9 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Thu, 25 Sep 2025 07:00:30 +0000 Subject: [PATCH 2/6] DAOS-17788 test: Use command_as_user to set root to user Handle rdb_pool_paths as a list instead of hard-coding the two paths separately. Use get_engine_values("scm_mount") to directly get the two pool paths. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_lost_all_rdb test_lost_majority_ps_replicas Signed-off-by: Makito Kano --- .../ftest/recovery/pool_list_consolidation.py | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/tests/ftest/recovery/pool_list_consolidation.py b/src/tests/ftest/recovery/pool_list_consolidation.py index 49aa87922a8..ab08323133b 100644 --- a/src/tests/ftest/recovery/pool_list_consolidation.py +++ b/src/tests/ftest/recovery/pool_list_consolidation.py @@ -11,7 +11,7 @@ from ClusterShell.NodeSet import NodeSet from general_utils import check_file_exists, report_errors from recovery_utils import wait_for_check_complete -from run_utils import run_remote +from run_utils import command_as_user, run_remote class PoolListConsolidationTest(TestWithServers): @@ -290,14 +290,16 @@ def test_lost_majority_ps_replicas(self): self.log_step("Remove //rdb-pool from two ranks.") rdb_pool_path = f"{self.server_managers[0].get_vos_path(pool)}/rdb-pool" - command = f"sudo rm {rdb_pool_path}" hosts = list(set(self.server_managers[0].ranks.values())) count = 0 for host in hosts: node = NodeSet(host) check_out = check_file_exists(hosts=node, filename=rdb_pool_path, sudo=True) if check_out[0]: - if not run_remote(log=self.log, hosts=node, command=command).passed: + remove_cmd_passed = run_remote( + log=self.log, hosts=node, command=f"rm {rdb_pool_path}", + user="root").passed + if not remove_cmd_passed: self.fail(f'Failed to remove {rdb_pool_path} on {host}') self.log.info("rm rdb-pool from %s", str(node)) count += 1 @@ -379,35 +381,26 @@ def test_lost_all_rdb(self): dmg_command.system_stop() self.log_step("Remove //rdb-pool from all ranks.") - engine_params = self.server_managers[0].manager.job.yaml.engine_params - scm_mount_0 = engine_params[0].get_value("scm_mount") - scm_mount_1 = engine_params[1].get_value("scm_mount") - rdb_pool_path_0 = f"{scm_mount_0}/{pool.uuid.lower()}/rdb-pool" - rdb_pool_path_1 = f"{scm_mount_1}/{pool.uuid.lower()}/rdb-pool" - self.log.info("rdb_pool_path_0 = %s", rdb_pool_path_0) - self.log.info("rdb_pool_path_1 = %s", rdb_pool_path_1) - rdb_pool_out_0 = check_file_exists( - hosts=self.hostlist_servers, filename=rdb_pool_path_0, sudo=True) - rdb_pool_out_1 = check_file_exists( - hosts=self.hostlist_servers, filename=rdb_pool_path_1, sudo=True) - if not rdb_pool_out_0[0] or not rdb_pool_out_1[0]: + scm_mounts = self.server_managers[0].manager.job.yaml.get_engine_values( + "scm_mount") + rdb_pool_paths = [f"{scm}/{pool.uuid.lower()}/rdb-pool" for scm in scm_mounts] + self.log.info("rdb_pool_paths: %s", rdb_pool_paths) + rdb_pool_exists = [check_file_exists( + self.hostlist_servers, path, sudo=True)[0] for path in rdb_pool_paths] + if not all(rdb_pool_exists): msg = ("MD-on-SSD cluster. Contents under mount point are removed by control plane " "after system stop.") self.log.info(msg) dmg_command.system_start() # return results in PASS. return - rdb_pool_path_list = [rdb_pool_path_0, rdb_pool_path_1] - for rdb_pool_path in rdb_pool_path_list: - command = f"sudo rm {rdb_pool_path}" - remove_result = run_remote(log=self.log, hosts=self.hostlist_servers, command=command) + for rdb_pool_path in rdb_pool_paths: + command = command_as_user(command=f"rm {rdb_pool_path}", user="root") + remove_result = run_remote( + log=self.log, hosts=self.hostlist_servers, command=command) if not remove_result.passed: - self.fail(f"Failed to remove {rdb_pool_path} from {self.hostlist_servers}") - success_nodes = remove_result.passed_hosts - if self.hostlist_servers != success_nodes: - msg = (f"Failed to remove rdb-pool! All = {self.hostlist_servers}, " - f"Success = {success_nodes}") - self.fail(msg) + self.fail( + f"Failed to remove {rdb_pool_path} from {remove_result.failed_hosts}") self.log_step("Run DAOS checker under kinds of mode.") errors = [] From 2b2a6315a182704b459ee5fe3116d4556043ad5a Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Thu, 25 Sep 2025 07:11:32 +0000 Subject: [PATCH 3/6] DAOS-17788 test: Use command_as_user that was missing Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_lost_all_rdb test_lost_majority_ps_replicas Signed-off-by: Makito Kano --- src/tests/ftest/recovery/pool_list_consolidation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/pool_list_consolidation.py b/src/tests/ftest/recovery/pool_list_consolidation.py index ab08323133b..a5fc9220fdf 100644 --- a/src/tests/ftest/recovery/pool_list_consolidation.py +++ b/src/tests/ftest/recovery/pool_list_consolidation.py @@ -290,6 +290,7 @@ def test_lost_majority_ps_replicas(self): self.log_step("Remove //rdb-pool from two ranks.") rdb_pool_path = f"{self.server_managers[0].get_vos_path(pool)}/rdb-pool" + command = command_as_user(command=f"rm {rdb_pool_path}", user="root") hosts = list(set(self.server_managers[0].ranks.values())) count = 0 for host in hosts: @@ -297,8 +298,7 @@ def test_lost_majority_ps_replicas(self): check_out = check_file_exists(hosts=node, filename=rdb_pool_path, sudo=True) if check_out[0]: remove_cmd_passed = run_remote( - log=self.log, hosts=node, command=f"rm {rdb_pool_path}", - user="root").passed + log=self.log, hosts=node, command=command).passed if not remove_cmd_passed: self.fail(f'Failed to remove {rdb_pool_path} on {host}') self.log.info("rm rdb-pool from %s", str(node)) From 3bbdb1d5dadaae07814e2ba4c0a7ec8c64288b6d Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Sun, 28 Sep 2025 00:00:08 +0000 Subject: [PATCH 4/6] DAOS-17788 test: Revert change in test_lost_majority_ps_replicas The test is failing with different issue. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_lost_all_rdb Signed-off-by: Makito Kano --- src/tests/ftest/recovery/pool_list_consolidation.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/recovery/pool_list_consolidation.py b/src/tests/ftest/recovery/pool_list_consolidation.py index a5fc9220fdf..41be76bd5cf 100644 --- a/src/tests/ftest/recovery/pool_list_consolidation.py +++ b/src/tests/ftest/recovery/pool_list_consolidation.py @@ -290,16 +290,14 @@ def test_lost_majority_ps_replicas(self): self.log_step("Remove //rdb-pool from two ranks.") rdb_pool_path = f"{self.server_managers[0].get_vos_path(pool)}/rdb-pool" - command = command_as_user(command=f"rm {rdb_pool_path}", user="root") + command = f"sudo rm {rdb_pool_path}" hosts = list(set(self.server_managers[0].ranks.values())) count = 0 for host in hosts: node = NodeSet(host) check_out = check_file_exists(hosts=node, filename=rdb_pool_path, sudo=True) if check_out[0]: - remove_cmd_passed = run_remote( - log=self.log, hosts=node, command=command).passed - if not remove_cmd_passed: + if not run_remote(log=self.log, hosts=node, command=command).passed: self.fail(f'Failed to remove {rdb_pool_path} on {host}') self.log.info("rm rdb-pool from %s", str(node)) count += 1 From 0798ca1e85e1f14c39908ece60b6459ebc87fc5b Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Sun, 28 Sep 2025 04:57:33 +0000 Subject: [PATCH 5/6] DAOS-17788 test: Update comment Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_lost_all_rdb Signed-off-by: Makito Kano --- src/tests/ftest/recovery/pool_list_consolidation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/recovery/pool_list_consolidation.py b/src/tests/ftest/recovery/pool_list_consolidation.py index 41be76bd5cf..820abc97a47 100644 --- a/src/tests/ftest/recovery/pool_list_consolidation.py +++ b/src/tests/ftest/recovery/pool_list_consolidation.py @@ -361,7 +361,7 @@ def test_lost_all_rdb(self): 2. Stop servers. 3. Remove //rdb-pool from all ranks. 4. Run DAOS checker under kinds of mode. - 5. Check that the pool does not appear with dmg pool list. + 5. Check that the pool does not appear with "dmg pool list". 6. Verify that the pool directory was removed from the mount point. Jira ID: DAOS-12067 From e6c2b88248718d4db17f8874245031b24d28fa04 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Mon, 29 Sep 2025 05:10:55 +0000 Subject: [PATCH 6/6] DAOS-17788 test: Update comment. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_lost_all_rdb Signed-off-by: Makito Kano --- src/tests/ftest/recovery/pool_list_consolidation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/recovery/pool_list_consolidation.py b/src/tests/ftest/recovery/pool_list_consolidation.py index 820abc97a47..41be76bd5cf 100644 --- a/src/tests/ftest/recovery/pool_list_consolidation.py +++ b/src/tests/ftest/recovery/pool_list_consolidation.py @@ -361,7 +361,7 @@ def test_lost_all_rdb(self): 2. Stop servers. 3. Remove //rdb-pool from all ranks. 4. Run DAOS checker under kinds of mode. - 5. Check that the pool does not appear with "dmg pool list". + 5. Check that the pool does not appear with dmg pool list. 6. Verify that the pool directory was removed from the mount point. Jira ID: DAOS-12067