From e44ee366ca4595a8780c3f50035cb97c0a5ad224 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 23 Dec 2025 13:50:39 -0500 Subject: [PATCH 01/78] DAOS-17519 test: Automate dlck testing (basic/fault_injection) Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 48 +++++++++ src/tests/ftest/recovery/dlck_basic.yaml | 22 ++++ src/tests/ftest/recovery/dlck_basic_fault.py | 91 ++++++++++++++++ .../ftest/recovery/dlck_basic_fault.yaml | 49 +++++++++ src/tests/ftest/util/dlck_utils.py | 89 ++++++++++++++++ src/tests/ftest/util/fault_config_utils.py | 100 ++++++++++++++++++ 6 files changed, 399 insertions(+) create mode 100644 src/tests/ftest/recovery/dlck_basic.py create mode 100644 src/tests/ftest/recovery/dlck_basic.yaml create mode 100644 src/tests/ftest/recovery/dlck_basic_fault.py create mode 100644 src/tests/ftest/recovery/dlck_basic_fault.yaml create mode 100644 src/tests/ftest/util/dlck_utils.py diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py new file mode 100644 index 00000000000..fac944edea0 --- /dev/null +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -0,0 +1,48 @@ +""" + (C) Copyright 2025 Hewlett Packard Enterprise Development LP + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os + +from apricot import TestWithServers +from dlck_utils import DlckCommand + +class DlckBasicTest(TestWithServers): + """Test class for dlck command line utility. + + :avocado: recursive + """ + def test_dlck_basic_test(self): + """Basic Test: Run 'dlck' command + + :avocado: tags=all,daily_regression + :avocado: tags=hw,medium + :avocado: tags=recovery + :avocado: tags=DlckBasicTest,dlck_cmd,test_dlck_basic + """ + errors = [] + dmg = self.get_dmg_command() + self.add_pool() + pool_uuids = dmg.get_pool_list_uuids(no_query=True) + scm_mount = self.server_managers[0].get_config_value("scm_mount") + if self.server_managers[0].manager.job.using_control_metadata: + log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) + control_metadata_dir = os.path.join(log_dir, "control_metadata") + nvme_conf=os.path.join(control_metadata_dir, "daos_nvme.conf") + dmg.system_stop() + host = self.server_managers[0].hosts[0:1] + if self.server_managers[0].manager.job.using_control_metadata: + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, + storage_mount=scm_mount) + else: + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount) + result = dlck_cmd.run() + if not result.passed: + errors.append(f"dlck failed on {result.failed_hosts}") + self.log.info("dlck basic test output:\n%s", result) + dmg.system_start() + if errors: + self.fail("Errors detected:\n{}".format("\n".join(errors))) + self.log.info("dlck basic test passed with no errors") + diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml new file mode 100644 index 00000000000..ce2831b245e --- /dev/null +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -0,0 +1,22 @@ +hosts: + test_servers: 1 + test_clients: 1 + +timeout: 500 + +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + targets: 4 + storage: auto + 1: + targets: 4 + storage: auto + +setup: + start_servers_once: False + +pool: + size: 50% diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py new file mode 100644 index 00000000000..38beb045bf9 --- /dev/null +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -0,0 +1,91 @@ +""" + (C) Copyright 2025 Hewlett Packard Enterprise Development LP + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os + +from apricot import TestWithServers +from dlck_utils import DlckCommand +from run_utils import run_remote +from file_utils import distribute_files +from fault_config_utils import FaultInjection + +class DlckBasicFaultTest(TestWithServers): + """Test class for dlck command line utility. + + :avocado: recursive + """ + def test_dlck_basic_fault_test(self): + """Basic Fault Test: Run 'dlck' injecting basic faults. + + :avocado: tags=all,full_regression + :avocado: tags=hw,medium + :avocado: tags=recovery + :avocado: tags=DlckBasicFaultTest,dlck_cmd,test_dlck_basic_fault + """ + errors = [] + faults_dict = {} + faults_object = FaultInjection() + faults_dict = faults_object.get_faults_dict() + fault_list = self.params.get("fault_list", '/run/dlck_test_additional_faults/*') + dmg = self.get_dmg_command() + self.add_pool() + pool_uuids = dmg.get_pool_list_uuids(no_query=True) + scm_mount = self.server_managers[0].get_config_value("scm_mount") + if self.server_managers[0].manager.job.using_control_metadata: + log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) + control_metadata_dir = os.path.join(log_dir, "control_metadata") + nvme_conf=os.path.join(control_metadata_dir, "daos_nvme.conf") + fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") + if fault_inject_file == "None set for now": + self.fail("D_FI_CONFIG environment variable not set, cannot run fault injection test") + env_str = "D_FI_CONFIG={} ".format(fault_inject_file) + self.log.info("Fault injection file contents") + cmd = "cat {}".format(fault_inject_file) + host = self.server_managers[0].hosts[0:1] + run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30) + dmg.system_stop() + # Run the testing with the first fault which is injected at the beginning of the test. + if self.server_managers[0].manager.job.using_control_metadata: + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, + storage_mount=scm_mount, env_str=env_str) + else: + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, env_str=env_str) + result = dlck_cmd.run() + if not result.passed: + errors.append(f"dlck failed on {result.failed_hosts}") + self.log.info("dlck basic test output:\n%s", result) + # Now, run the other fault injection flags without rebooting or creating any new pools. + # Rebooting the servers or creating the new pools will result in injectung fault in the wrong + # test code. Fault injections should done only for the dlck alone. + for test_fault in fault_list: + with open(fault_inject_file, 'w') as f: + f.write(f"fault_config:\n") + count = 0 + for key, value in faults_dict[test_fault].items(): + if count == 0: + f.write(f"- {key}: \'{value}\'\n") + else: + f.write(f" {key}: \'{value}\'\n") + count += 1 + f.close() + self.log.info("Reading the updated fault injection file contents") + with open(fault_inject_file, 'r') as f: + file_data = f.read() + self.log.info("\n%s", file_data) + f.close() + distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) + #faults_object.copy_fault_files(self.log, self.hostlist_servers) + if self.server_managers[0].manager.job.using_control_metadata: + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, + storage_mount=scm_mount, env_str=env_str) + else: + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, env_str=env_str) + result = dlck_cmd.run() + if not result.passed: + errors.append(f"dlck failed on {result.failed_hosts}") + self.log.info("dlck basic test output:\n%s", result) + dmg.system_start() + if not errors: + self.fail("No Errors detected:\n{}".format("\n".join(errors))) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml new file mode 100644 index 00000000000..b3ae5895f6b --- /dev/null +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -0,0 +1,49 @@ +hosts: + test_servers: 1 + test_clients: 1 + +timeout: 500 + +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + targets: 4 + storage: auto + 1: + targets: 4 + storage: auto + +setup: + start_servers_once: True + +pool: + size: 50% + +dlck: + scm_mount: /mnt/daos0 + +faults: !mux + create_log_dir: + fault_list: + - DLCK_FAULT_CREATE_LOG_DIR + +dlck_test_additional_faults: + fault_list: + - DLCK_FAULT_CREATE_POOL_DIR + - DLCK_FAULT_ENGINE_START + - DLCK_FAULT_ENGINE_EXEC + - DLCK_FAULT_ENGINE_JOIN + - DLCK_FAULT_ENGINE_STOP + - DAOS_FAULT_POOL_NVME_HEALTH + - DAOS_FAULT_POOL_OPEN_BIO + - DAOS_FAULT_POOL_OPEN_UMEM + - DAOS_FAULT_POOL_OPEN_MAGIC + - DAOS_FAULT_POOL_OPEN_VERSION + - DAOS_FAULT_POOL_OPEN_UUID + - DAOS_FAULT_BTREE_OPEN_INV_CLASS + - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS + - DAOS_FAULT_BTREE_FEATURES + - DAOS_FAULT_POOL_EXT_PADDING + - DAOS_FAULT_POOL_EXT_RESERVED diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py new file mode 100644 index 00000000000..894c65eb4e7 --- /dev/null +++ b/src/tests/ftest/util/dlck_utils.py @@ -0,0 +1,89 @@ +""" + (C) Copyright 2025 Hewlett Packard Enterprise Development LP + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" + +from command_utils_base import CommandWithParameters, FormattedParameter +from run_utils import run_remote + + +class DlckCommand(CommandWithParameters): + """Defines the basic structures of dlck command.""" + + def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mount=None, + verbose=True, timeout=None, sudo=True, env_str=None): + """Constructor that sets the common variables for sub-commands. + + Args: + server_host (NodeSet): Server host to run the command. + path (str): path to the dlck command. + pool_uuid (str, optional): Pool UUID. Defaults to None. + nvme_conf (str, optional): NVMe config file path. Defaults to None. + storage_mount (str, optional): Storage mount point. Defaults to None. + verbose (bool, optional): Display command output in run. + Defaults to True. + timeout (int, optional): Command timeout (sec) used in run. Defaults to + None. + sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. + env_str (str, optional): Environment variable string to prepend to command. + + """ + super().__init__("/run/dlck/*", "dlck", path) + # Pass environment variable string + self.env_str = "" + if env_str is not None: + self.env_str = str(env_str) + + + # We need to run with sudo. + self.sudo = sudo + + self.host = server_host + + # Members needed for run(). + self.verbose = verbose + self.timeout = timeout + + # Pool UUID. (--file pool_uuid[,target_id]) + if pool_uuid: + self.pool_uuid = FormattedParameter("--file={}", pool_uuid) + + # NVMe config file path. (--nvme_cont nvme_conf_path) + if nvme_conf: + self.nvme_conf = FormattedParameter("--nvme={}", nvme_conf) + + # Storage mount point. (--storage storage_mount) + if storage_mount: + self.storage_mount = FormattedParameter("--storage={}", storage_mount) + + def __str__(self): + """Return the command with all of its defined parameters as a string. + + Returns: + str: the command with all the defined parameters + + """ + value = super().__str__() + if self.sudo: + value = " ".join(["sudo -E -n", value]) + return value + + def run(self): + """Run the dlck command. + + Args: + host (NodeSet): Host(s) on which to run the command. + command (str): Environment Variable string + dlck sub-command to run. + verbose (bool, optional): Display command output in run. + Defaults to True. + timeout (int, optional): Command timeout (sec) used in run. Defaults to + None. + + Returns: + CommandResult: groups of command results from the same hosts with the same return status + + """ + return run_remote( + self.log, self.host, command=self.env_str + str(self), verbose=self.verbose, + timeout=self.timeout) diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index 5dd3071271f..f4c6188e5f4 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -1,5 +1,6 @@ """ (C) Copyright 2019-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -235,6 +236,99 @@ 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, + 'DLCK_FAULT_CREATE_LOG_DIR': { + 'id': '131328', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_CREATE_POOL_DIR': { + 'id': '131329', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_START': { + 'id': '131330', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_EXEC': { + 'id': '131331', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_JOIN': { + 'id': '131332', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_STOP': { + 'id': '131333', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '1', + 'max_faults': '1'}, + 'DAOS_FAULT_POOL_NVME_HEALTH': { + 'id': '131584', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '2'}, + 'DAOS_FAULT_POOL_OPEN_BIO': { + 'id': '131585', + 'probability_x': '100', + 'probability_y': '100'}, + 'DAOS_FAULT_POOL_OPEN_UMEM': { + 'id': '131586', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '2'}, + 'DAOS_FAULT_POOL_OPEN_MAGIC': { + 'id': '131587', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '2'}, + 'DAOS_FAULT_POOL_OPEN_VERSION': { + 'id': '131588', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '2'}, + 'DAOS_FAULT_POOL_OPEN_UUID': { + 'id': '131589', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '2'}, + 'DAOS_FAULT_BTREE_OPEN_INV_CLASS': { + 'id': '131590', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '28', + 'max_faults': '1'}, + 'DAOS_FAULT_BTREE_OPEN_UNREG_CLASS': { + 'id': '131591', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '29', + 'max_faults': '1'}, + 'DAOS_FAULT_BTREE_FEATURES': { + 'id': '131592', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '28', + 'max_faults': '1'}, + 'DAOS_FAULT_POOL_EXT_PADDING': { + 'id': '131593', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '1'}, + 'DAOS_FAULT_POOL_EXT_RESERVED': { + 'id': '131594', + 'probability_x': '100', + 'probability_y': '100', + 'interval': '1'}, } @@ -255,6 +349,12 @@ def __init__(self): self._test_dir = None self._fault_list = [] + def get_faults_dict(self): + """Get the predefined fault dictionary. + Returns: dict: The fault dictionary + """ + return FAULTS + def write_fault_file(self, on_the_fly_fault=None): """Write out a fault injection config file. From 1dfe0ef4a86989bf1d8ca254a8b3a488165c458a Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 23 Dec 2025 16:34:11 -0500 Subject: [PATCH 02/78] DAOS-17519 test: Fix linting issues. Test-tag: DlckBasicTest DlckBasicFaultTest Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 16 ++-- src/tests/ftest/recovery/dlck_basic.yaml | 4 +- src/tests/ftest/recovery/dlck_basic_fault.py | 79 ++++++++++--------- .../ftest/recovery/dlck_basic_fault.yaml | 28 +++---- src/tests/ftest/util/dlck_utils.py | 10 +-- 5 files changed, 68 insertions(+), 69 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index fac944edea0..632e4c0c0dd 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -8,6 +8,7 @@ from apricot import TestWithServers from dlck_utils import DlckCommand + class DlckBasicTest(TestWithServers): """Test class for dlck command line utility. @@ -27,22 +28,21 @@ def test_dlck_basic_test(self): pool_uuids = dmg.get_pool_list_uuids(no_query=True) scm_mount = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: - log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) - control_metadata_dir = os.path.join(log_dir, "control_metadata") - nvme_conf=os.path.join(control_metadata_dir, "daos_nvme.conf") + log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) + control_metadata_dir = os.path.join(log_dir, "control_metadata") + nvme_conf=os.path.join(control_metadata_dir, "daos_nvme.conf") dmg.system_stop() host = self.server_managers[0].hosts[0:1] if self.server_managers[0].manager.job.using_control_metadata: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, - storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, + storage_mount=scm_mount) else: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount) result = dlck_cmd.run() if not result.passed: - errors.append(f"dlck failed on {result.failed_hosts}") + errors.append(f"dlck failed on {result.failed_hosts}") self.log.info("dlck basic test output:\n%s", result) dmg.system_start() if errors: self.fail("Errors detected:\n{}".format("\n".join(errors))) self.log.info("dlck basic test passed with no errors") - diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index ce2831b245e..93773b99d1a 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -9,10 +9,10 @@ server_config: engines_per_host: 1 engines: 0: - targets: 4 + targets: 4 storage: auto 1: - targets: 4 + targets: 4 storage: auto setup: diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 38beb045bf9..325e15ebfcb 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -7,9 +7,10 @@ from apricot import TestWithServers from dlck_utils import DlckCommand -from run_utils import run_remote -from file_utils import distribute_files from fault_config_utils import FaultInjection +from file_utils import distribute_files +from run_utils import run_remote + class DlckBasicFaultTest(TestWithServers): """Test class for dlck command line utility. @@ -34,12 +35,12 @@ def test_dlck_basic_fault_test(self): pool_uuids = dmg.get_pool_list_uuids(no_query=True) scm_mount = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: - log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) - control_metadata_dir = os.path.join(log_dir, "control_metadata") - nvme_conf=os.path.join(control_metadata_dir, "daos_nvme.conf") + log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) + control_metadata_dir = os.path.join(log_dir, "control_metadata") + nvme_conf=os.path.join(control_metadata_dir, "daos_nvme.conf") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file == "None set for now": - self.fail("D_FI_CONFIG environment variable not set, cannot run fault injection test") + self.fail("D_FI_CONFIG environment variable not set, cannot run fault injection test") env_str = "D_FI_CONFIG={} ".format(fault_inject_file) self.log.info("Fault injection file contents") cmd = "cat {}".format(fault_inject_file) @@ -48,44 +49,46 @@ def test_dlck_basic_fault_test(self): dmg.system_stop() # Run the testing with the first fault which is injected at the beginning of the test. if self.server_managers[0].manager.job.using_control_metadata: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, - storage_mount=scm_mount, env_str=env_str) + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, + storage_mount=scm_mount, env_str=env_str) else: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, env_str=env_str) + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, + env_str=env_str) result = dlck_cmd.run() if not result.passed: - errors.append(f"dlck failed on {result.failed_hosts}") + errors.append(f"dlck failed on {result.failed_hosts}") self.log.info("dlck basic test output:\n%s", result) # Now, run the other fault injection flags without rebooting or creating any new pools. - # Rebooting the servers or creating the new pools will result in injectung fault in the wrong - # test code. Fault injections should done only for the dlck alone. + # Rebooting the servers or creating the new pools will result in injectung fault in + # the wrong test code. Fault injections should done only for the dlck alone. for test_fault in fault_list: - with open(fault_inject_file, 'w') as f: - f.write(f"fault_config:\n") - count = 0 - for key, value in faults_dict[test_fault].items(): - if count == 0: - f.write(f"- {key}: \'{value}\'\n") - else: - f.write(f" {key}: \'{value}\'\n") - count += 1 - f.close() - self.log.info("Reading the updated fault injection file contents") - with open(fault_inject_file, 'r') as f: - file_data = f.read() - self.log.info("\n%s", file_data) - f.close() - distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) - #faults_object.copy_fault_files(self.log, self.hostlist_servers) - if self.server_managers[0].manager.job.using_control_metadata: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, - storage_mount=scm_mount, env_str=env_str) - else: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, env_str=env_str) - result = dlck_cmd.run() - if not result.passed: - errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info("dlck basic test output:\n%s", result) + with open(fault_inject_file, 'w') as f: + f.write(f"fault_config:\n") + count = 0 + for key, value in faults_dict[test_fault].items(): + if count == 0: + f.write(f"- {key}: \'{value}\'\n") + else: + f.write(f" {key}: \'{value}\'\n") + count += 1 + f.close() + self.log.info("Reading the updated fault injection file contents") + with open(fault_inject_file, 'r') as f: + file_data = f.read() + self.log.info("\n%s", file_data) + f.close() + distribute_files(self.log, self.hostlist_servers, fault_inject_file, + fault_inject_file) + if self.server_managers[0].manager.job.using_control_metadata: + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, + storage_mount=scm_mount, env_str=env_str) + else: + dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, + env_str=env_str) + result = dlck_cmd.run() + if not result.passed: + errors.append(f"dlck failed on {result.failed_hosts}") + self.log.info("dlck basic test output:\n%s", result) dmg.system_start() if not errors: self.fail("No Errors detected:\n{}".format("\n".join(errors))) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index b3ae5895f6b..7be32164b2d 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -9,10 +9,10 @@ server_config: engines_per_host: 1 engines: 0: - targets: 4 + targets: 4 storage: auto 1: - targets: 4 + targets: 4 storage: auto setup: @@ -33,17 +33,17 @@ dlck_test_additional_faults: fault_list: - DLCK_FAULT_CREATE_POOL_DIR - DLCK_FAULT_ENGINE_START - - DLCK_FAULT_ENGINE_EXEC + - DLCK_FAULT_ENGINE_EXEC - DLCK_FAULT_ENGINE_JOIN - - DLCK_FAULT_ENGINE_STOP - - DAOS_FAULT_POOL_NVME_HEALTH - - DAOS_FAULT_POOL_OPEN_BIO - - DAOS_FAULT_POOL_OPEN_UMEM - - DAOS_FAULT_POOL_OPEN_MAGIC - - DAOS_FAULT_POOL_OPEN_VERSION - - DAOS_FAULT_POOL_OPEN_UUID - - DAOS_FAULT_BTREE_OPEN_INV_CLASS - - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS - - DAOS_FAULT_BTREE_FEATURES - - DAOS_FAULT_POOL_EXT_PADDING + - DLCK_FAULT_ENGINE_STOP + - DAOS_FAULT_POOL_NVME_HEALTH + - DAOS_FAULT_POOL_OPEN_BIO + - DAOS_FAULT_POOL_OPEN_UMEM + - DAOS_FAULT_POOL_OPEN_MAGIC + - DAOS_FAULT_POOL_OPEN_VERSION + - DAOS_FAULT_POOL_OPEN_UUID + - DAOS_FAULT_BTREE_OPEN_INV_CLASS + - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS + - DAOS_FAULT_BTREE_FEATURES + - DAOS_FAULT_POOL_EXT_PADDING - DAOS_FAULT_POOL_EXT_RESERVED diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 894c65eb4e7..8a69297f33d 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -7,7 +7,7 @@ from command_utils_base import CommandWithParameters, FormattedParameter from run_utils import run_remote - + class DlckCommand(CommandWithParameters): """Defines the basic structures of dlck command.""" @@ -27,14 +27,12 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo None. sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. env_str (str, optional): Environment variable string to prepend to command. - """ super().__init__("/run/dlck/*", "dlck", path) # Pass environment variable string self.env_str = "" if env_str is not None: - self.env_str = str(env_str) - + self.env_str = str(env_str) # We need to run with sudo. self.sudo = sudo @@ -68,10 +66,9 @@ def __str__(self): if self.sudo: value = " ".join(["sudo -E -n", value]) return value - + def run(self): """Run the dlck command. - Args: host (NodeSet): Host(s) on which to run the command. command (str): Environment Variable string + dlck sub-command to run. @@ -82,7 +79,6 @@ def run(self): Returns: CommandResult: groups of command results from the same hosts with the same return status - """ return run_remote( self.log, self.host, command=self.env_str + str(self), verbose=self.verbose, From aa8ede9db5b185110f8eb5e0495a73db8fbb3c31 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 12 Mar 2026 12:27:10 -0400 Subject: [PATCH 03/78] DAOS-17519 test: Some of the review updates. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 2 +- src/tests/ftest/recovery/dlck_basic_fault.py | 15 +++--- .../ftest/recovery/dlck_basic_fault.yaml | 5 +- src/tests/ftest/util/dlck_utils.py | 5 +- src/tests/ftest/util/fault_config_utils.py | 49 +++---------------- 5 files changed, 20 insertions(+), 56 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 632e4c0c0dd..8d554f04500 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 325e15ebfcb..dfafcd01626 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -37,7 +37,10 @@ def test_dlck_basic_fault_test(self): if self.server_managers[0].manager.job.using_control_metadata: log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") - nvme_conf=os.path.join(control_metadata_dir, "daos_nvme.conf") + engine_path_dir = os.path.join(control_metadata_dir, "engine0") + nvme_conf=os.path.join(engine_path_dir, "daos_nvme.conf") + self.log.info(f"directory: {os.listdir(control_metadata_dir)}") + self.log.info(f"directory: {os.listdir(engine_path_dir)}") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file == "None set for now": self.fail("D_FI_CONFIG environment variable not set, cannot run fault injection test") @@ -59,11 +62,11 @@ def test_dlck_basic_fault_test(self): errors.append(f"dlck failed on {result.failed_hosts}") self.log.info("dlck basic test output:\n%s", result) # Now, run the other fault injection flags without rebooting or creating any new pools. - # Rebooting the servers or creating the new pools will result in injectung fault in + # Rebooting the servers or creating the new pools will result in injecting fault in # the wrong test code. Fault injections should done only for the dlck alone. for test_fault in fault_list: with open(fault_inject_file, 'w') as f: - f.write(f"fault_config:\n") + f.write("fault_config:\n") count = 0 for key, value in faults_dict[test_fault].items(): if count == 0: @@ -71,12 +74,10 @@ def test_dlck_basic_fault_test(self): else: f.write(f" {key}: \'{value}\'\n") count += 1 - f.close() self.log.info("Reading the updated fault injection file contents") with open(fault_inject_file, 'r') as f: file_data = f.read() self.log.info("\n%s", file_data) - f.close() distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) if self.server_managers[0].manager.job.using_control_metadata: @@ -88,7 +89,7 @@ def test_dlck_basic_fault_test(self): result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info("dlck basic test output:\n%s", result) + self.log.info(f"dlck basic test output: {result}") dmg.system_start() if not errors: self.fail("No Errors detected:\n{}".format("\n".join(errors))) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 7be32164b2d..4c48186841b 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -11,9 +11,6 @@ server_config: 0: targets: 4 storage: auto - 1: - targets: 4 - storage: auto setup: start_servers_once: True @@ -24,7 +21,7 @@ pool: dlck: scm_mount: /mnt/daos0 -faults: !mux +faults: create_log_dir: fault_list: - DLCK_FAULT_CREATE_LOG_DIR diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 8a69297f33d..1cf029e364c 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -47,9 +47,9 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo if pool_uuid: self.pool_uuid = FormattedParameter("--file={}", pool_uuid) - # NVMe config file path. (--nvme_cont nvme_conf_path) + # NVMe config file path. (--nvme nvme_conf) if nvme_conf: - self.nvme_conf = FormattedParameter("--nvme={}", nvme_conf) + self.nvme = FormattedParameter("--nvme={}", nvme_conf) # Storage mount point. (--storage storage_mount) if storage_mount: @@ -60,7 +60,6 @@ def __str__(self): Returns: str: the command with all the defined parameters - """ value = super().__str__() if self.sudo: diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index f4c6188e5f4..93b014a11ce 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -238,96 +238,63 @@ 'max_faults': '1'}, 'DLCK_FAULT_CREATE_LOG_DIR': { 'id': '131328', - 'probability_x': '100', - 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, 'DLCK_FAULT_CREATE_POOL_DIR': { 'id': '131329', - 'probability_x': '100', - 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, 'DLCK_FAULT_ENGINE_START': { 'id': '131330', - 'probability_x': '100', - 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, 'DLCK_FAULT_ENGINE_EXEC': { 'id': '131331', - 'probability_x': '100', - 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, 'DLCK_FAULT_ENGINE_JOIN': { 'id': '131332', - 'probability_x': '100', - 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, 'DLCK_FAULT_ENGINE_STOP': { 'id': '131333', - 'probability_x': '100', - 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, 'DAOS_FAULT_POOL_NVME_HEALTH': { 'id': '131584', - 'probability_x': '100', - 'probability_y': '100', - 'interval': '2'}, + 'interval': '2', # skip sys_db + 'max_faults': '1'}, 'DAOS_FAULT_POOL_OPEN_BIO': { - 'id': '131585', - 'probability_x': '100', - 'probability_y': '100'}, + 'id': '131585'}, 'DAOS_FAULT_POOL_OPEN_UMEM': { 'id': '131586', - 'probability_x': '100', - 'probability_y': '100', - 'interval': '2'}, + 'interval': '2'}, # skip sys_db 'DAOS_FAULT_POOL_OPEN_MAGIC': { 'id': '131587', - 'probability_x': '100', - 'probability_y': '100', - 'interval': '2'}, + 'interval': '2'}, # skip sys_db 'DAOS_FAULT_POOL_OPEN_VERSION': { 'id': '131588', - 'probability_x': '100', - 'probability_y': '100', - 'interval': '2'}, + 'interval': '2'}, # skip sys_db 'DAOS_FAULT_POOL_OPEN_UUID': { 'id': '131589', - 'probability_x': '100', - 'probability_y': '100', - 'interval': '2'}, + 'interval': '2'}, # skip sys_db 'DAOS_FAULT_BTREE_OPEN_INV_CLASS': { 'id': '131590', - 'probability_x': '100', - 'probability_y': '100', - 'interval': '28', + 'interval': '28', # containers tree fine-tuned 'max_faults': '1'}, 'DAOS_FAULT_BTREE_OPEN_UNREG_CLASS': { 'id': '131591', - 'probability_x': '100', - 'probability_y': '100', 'interval': '29', 'max_faults': '1'}, 'DAOS_FAULT_BTREE_FEATURES': { 'id': '131592', - 'probability_x': '100', - 'probability_y': '100', 'interval': '28', 'max_faults': '1'}, 'DAOS_FAULT_POOL_EXT_PADDING': { 'id': '131593', - 'probability_x': '100', - 'probability_y': '100', 'interval': '1'}, 'DAOS_FAULT_POOL_EXT_RESERVED': { 'id': '131594', - 'probability_x': '100', - 'probability_y': '100', 'interval': '1'}, } From 0cea3541652effb30439780ca35bd777cb68a891 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 31 Mar 2026 11:38:18 -0400 Subject: [PATCH 04/78] DAOS-17519 test: Fix Linting issues. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 14 +++++++++----- src/tests/ftest/recovery/dlck_basic_fault.py | 18 ++++++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 8d554f04500..e61683fd4db 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2026 Hewlett Packard Enterprise Development LP + Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -24,13 +24,16 @@ def test_dlck_basic_test(self): """ errors = [] dmg = self.get_dmg_command() + self.log_step("Create a pool to run dlck") self.add_pool() + self.log_step("Setup dlck parameters to run dlck command") pool_uuids = dmg.get_pool_list_uuids(no_query=True) scm_mount = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") - nvme_conf=os.path.join(control_metadata_dir, "daos_nvme.conf") + engine_path_dir = os.path.join(control_metadata_dir, "engine0") + nvme_conf=os.path.join(engine_path_dir, "daos_nvme.conf") dmg.system_stop() host = self.server_managers[0].hosts[0:1] if self.server_managers[0].manager.job.using_control_metadata: @@ -38,11 +41,12 @@ def test_dlck_basic_test(self): storage_mount=scm_mount) else: dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount) + self.log_step("Run dlck command to check the health of the pool and storage") result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info("dlck basic test output:\n%s", result) + self.log.info(f"dlck basic test output: %s\n", result) dmg.system_start() if errors: - self.fail("Errors detected:\n{}".format("\n".join(errors))) - self.log.info("dlck basic test passed with no errors") + self.fail("Errors detected:\n {}".format("\n".join(errors))) + self.log.info(f"dlck basic test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index dfafcd01626..0e95ff38e71 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2026 Hewlett Packard Enterprise Development LP + Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -39,14 +39,12 @@ def test_dlck_basic_fault_test(self): control_metadata_dir = os.path.join(log_dir, "control_metadata") engine_path_dir = os.path.join(control_metadata_dir, "engine0") nvme_conf=os.path.join(engine_path_dir, "daos_nvme.conf") - self.log.info(f"directory: {os.listdir(control_metadata_dir)}") - self.log.info(f"directory: {os.listdir(engine_path_dir)}") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file == "None set for now": - self.fail("D_FI_CONFIG environment variable not set, cannot run fault injection test") + self.fail(f"D_FI_CONFIG environment variable not set, cannot run fault injection test") env_str = "D_FI_CONFIG={} ".format(fault_inject_file) - self.log.info("Fault injection file contents") - cmd = "cat {}".format(fault_inject_file) + self.log.info(f"Fault injection file contents") + cmd = "cat {}".format(fault_inject_file) host = self.server_managers[0].hosts[0:1] run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30) dmg.system_stop() @@ -60,7 +58,7 @@ def test_dlck_basic_fault_test(self): result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info("dlck basic test output:\n%s", result) + self.log.info(f"dlck basic test output: %s \n", result) # Now, run the other fault injection flags without rebooting or creating any new pools. # Rebooting the servers or creating the new pools will result in injecting fault in # the wrong test code. Fault injections should done only for the dlck alone. @@ -72,12 +70,12 @@ def test_dlck_basic_fault_test(self): if count == 0: f.write(f"- {key}: \'{value}\'\n") else: - f.write(f" {key}: \'{value}\'\n") - count += 1 + f.write(f" {key}: \'{value}\'\n") + count += 1 self.log.info("Reading the updated fault injection file contents") with open(fault_inject_file, 'r') as f: file_data = f.read() - self.log.info("\n%s", file_data) + self.log.info(f"\n%s", file_data) distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) if self.server_managers[0].manager.job.using_control_metadata: From 585528a01eb21b8bc8844639e9b6ab03d187b08d Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 31 Mar 2026 12:22:17 -0400 Subject: [PATCH 05/78] DAOS-17519 test: Fix python/flake8 lint and copyright issues. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 12 ++++++------ src/tests/ftest/recovery/dlck_basic_fault.py | 16 ++++++++-------- src/tests/ftest/util/dlck_utils.py | 4 ++-- src/tests/ftest/util/fault_config_utils.py | 16 ++++++++-------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index e61683fd4db..02d53cb6206 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -14,7 +14,7 @@ class DlckBasicTest(TestWithServers): :avocado: recursive """ - def test_dlck_basic_test(self): + def test_dlck_basic(self): """Basic Test: Run 'dlck' command :avocado: tags=all,daily_regression @@ -33,20 +33,20 @@ def test_dlck_basic_test(self): log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") engine_path_dir = os.path.join(control_metadata_dir, "engine0") - nvme_conf=os.path.join(engine_path_dir, "daos_nvme.conf") + nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") dmg.system_stop() host = self.server_managers[0].hosts[0:1] if self.server_managers[0].manager.job.using_control_metadata: dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, - storage_mount=scm_mount) + storage_mount=scm_mount) else: dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount) self.log_step("Run dlck command to check the health of the pool and storage") result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info(f"dlck basic test output: %s\n", result) + self.log.info(f"dlck basic test output: {result}\n") dmg.system_start() if errors: - self.fail("Errors detected:\n {}".format("\n".join(errors))) - self.log.info(f"dlck basic test passed with no errors") + self.fail(f"Errors detected: {errors} \n") + self.log.info("dlck basic test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 0e95ff38e71..a7b589af882 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -17,7 +17,7 @@ class DlckBasicFaultTest(TestWithServers): :avocado: recursive """ - def test_dlck_basic_fault_test(self): + def test_dlck_basic_fault(self): """Basic Fault Test: Run 'dlck' injecting basic faults. :avocado: tags=all,full_regression @@ -38,12 +38,12 @@ def test_dlck_basic_fault_test(self): log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") engine_path_dir = os.path.join(control_metadata_dir, "engine0") - nvme_conf=os.path.join(engine_path_dir, "daos_nvme.conf") + nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file == "None set for now": - self.fail(f"D_FI_CONFIG environment variable not set, cannot run fault injection test") + self.fail("D_FI_CONFIG environment variable not set") env_str = "D_FI_CONFIG={} ".format(fault_inject_file) - self.log.info(f"Fault injection file contents") + self.log.info("Fault injection file contents") cmd = "cat {}".format(fault_inject_file) host = self.server_managers[0].hosts[0:1] run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30) @@ -51,14 +51,14 @@ def test_dlck_basic_fault_test(self): # Run the testing with the first fault which is injected at the beginning of the test. if self.server_managers[0].manager.job.using_control_metadata: dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, - storage_mount=scm_mount, env_str=env_str) + storage_mount=scm_mount, env_str=env_str) else: dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, env_str=env_str) result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info(f"dlck basic test output: %s \n", result) + self.log.info(f"dlck basic test output: {result} \n") # Now, run the other fault injection flags without rebooting or creating any new pools. # Rebooting the servers or creating the new pools will result in injecting fault in # the wrong test code. Fault injections should done only for the dlck alone. @@ -75,12 +75,12 @@ def test_dlck_basic_fault_test(self): self.log.info("Reading the updated fault injection file contents") with open(fault_inject_file, 'r') as f: file_data = f.read() - self.log.info(f"\n%s", file_data) + self.log.info(f"\n {file_data}") distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) if self.server_managers[0].manager.job.using_control_metadata: dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, - storage_mount=scm_mount, env_str=env_str) + storage_mount=scm_mount, env_str=env_str) else: dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, env_str=env_str) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 1cf029e364c..bf6f820b878 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -32,7 +32,7 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo # Pass environment variable string self.env_str = "" if env_str is not None: - self.env_str = str(env_str) + self.env_str = str(env_str) # We need to run with sudo. self.sudo = sudo diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index 93b014a11ce..4721ec691ac 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -1,6 +1,6 @@ """ - (C) Copyright 2019-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + Copyright 2019-2024 Intel Corporation. + Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -262,22 +262,22 @@ 'max_faults': '1'}, 'DAOS_FAULT_POOL_NVME_HEALTH': { 'id': '131584', - 'interval': '2', # skip sys_db - 'max_faults': '1'}, + 'interval': '2', # skip sys_db + 'max_faults': '1'}, 'DAOS_FAULT_POOL_OPEN_BIO': { 'id': '131585'}, 'DAOS_FAULT_POOL_OPEN_UMEM': { 'id': '131586', - 'interval': '2'}, # skip sys_db + 'interval': '2'}, # skip sys_db 'DAOS_FAULT_POOL_OPEN_MAGIC': { 'id': '131587', - 'interval': '2'}, # skip sys_db + 'interval': '2'}, # skip sys_db 'DAOS_FAULT_POOL_OPEN_VERSION': { 'id': '131588', - 'interval': '2'}, # skip sys_db + 'interval': '2'}, # skip sys_db 'DAOS_FAULT_POOL_OPEN_UUID': { 'id': '131589', - 'interval': '2'}, # skip sys_db + 'interval': '2'}, # skip sys_db 'DAOS_FAULT_BTREE_OPEN_INV_CLASS': { 'id': '131590', 'interval': '28', # containers tree fine-tuned From e4508aa71f513152bc5f49ea6789f4617f873512 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 22 May 2026 13:22:37 -0400 Subject: [PATCH 06/78] DAOS-17519 test: Update the control_meta_dir. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 3 ++- src/tests/ftest/recovery/dlck_basic_fault.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 02d53cb6206..33dd09fc99a 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -32,7 +32,8 @@ def test_dlck_basic(self): if self.server_managers[0].manager.job.using_control_metadata: log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") - engine_path_dir = os.path.join(control_metadata_dir, "engine0") + daos_control_dir = os.path.join(control_meta_dir, "daos_control") + engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") dmg.system_stop() host = self.server_managers[0].hosts[0:1] diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index a7b589af882..69ed2ca2336 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -37,7 +37,8 @@ def test_dlck_basic_fault(self): if self.server_managers[0].manager.job.using_control_metadata: log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") - engine_path_dir = os.path.join(control_metadata_dir, "engine0") + daos_control_dir = os.path.join(control_meta_dir, "daos_control") + engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file == "None set for now": From 10920ac0d05ba1d90f9fd684532b09b8a0d92f4b Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Sat, 23 May 2026 14:04:11 -0400 Subject: [PATCH 07/78] DAOS-17519 test: Fix typo on control_metadata_dir path. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 2 +- src/tests/ftest/recovery/dlck_basic_fault.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 33dd09fc99a..52aca880d22 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -32,7 +32,7 @@ def test_dlck_basic(self): if self.server_managers[0].manager.job.using_control_metadata: log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") - daos_control_dir = os.path.join(control_meta_dir, "daos_control") + daos_control_dir = os.path.join(control_metadata_dir, "daos_control") engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") dmg.system_stop() diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 69ed2ca2336..3751ebd09e3 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -37,7 +37,7 @@ def test_dlck_basic_fault(self): if self.server_managers[0].manager.job.using_control_metadata: log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") - daos_control_dir = os.path.join(control_meta_dir, "daos_control") + daos_control_dir = os.path.join(control_metadata_dir, "daos_control") engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") From 410376917d5c5ca3c7ef85ac64b43bc68372cee9 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 12 Aug 2026 14:25:22 -0400 Subject: [PATCH 08/78] DAOS-17519 test: Fix some of the pylint errors. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 4 ++-- src/tests/ftest/recovery/dlck_basic_fault.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 52aca880d22..2a88a17fb07 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -46,8 +46,8 @@ def test_dlck_basic(self): result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info(f"dlck basic test output: {result}\n") + self.log.info(f"dlck basic test output: %s \n", result) dmg.system_start() if errors: - self.fail(f"Errors detected: {errors} \n") + self.fail(f"Errors detected: %s \n", errors) self.log.info("dlck basic test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 3751ebd09e3..29647e013a5 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -59,7 +59,7 @@ def test_dlck_basic_fault(self): result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info(f"dlck basic test output: {result} \n") + self.log.info(f"dlck basic test output: %s \n", result) # Now, run the other fault injection flags without rebooting or creating any new pools. # Rebooting the servers or creating the new pools will result in injecting fault in # the wrong test code. Fault injections should done only for the dlck alone. @@ -76,7 +76,7 @@ def test_dlck_basic_fault(self): self.log.info("Reading the updated fault injection file contents") with open(fault_inject_file, 'r') as f: file_data = f.read() - self.log.info(f"\n {file_data}") + self.log.info(f"\n %s", file_data) distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) if self.server_managers[0].manager.job.using_control_metadata: @@ -87,8 +87,8 @@ def test_dlck_basic_fault(self): env_str=env_str) result = dlck_cmd.run() if not result.passed: - errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info(f"dlck basic test output: {result}") + errors.append(f"dlck failed on %s", result.failed_hosts) + self.log.info(f"dlck basic test output: %s", result) dmg.system_start() if not errors: self.fail("No Errors detected:\n{}".format("\n".join(errors))) From 219147229e75aae367355a00c58c6e73cec50a8d Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 12 Aug 2026 15:25:43 -0400 Subject: [PATCH 09/78] DAOS-17519 test: Fix more pylint issues. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 7 ++++--- src/tests/ftest/recovery/dlck_basic_fault.py | 13 +++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 2a88a17fb07..18f2578c5ab 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -7,6 +7,7 @@ from apricot import TestWithServers from dlck_utils import DlckCommand +from test_utils_pool import add_pool class DlckBasicTest(TestWithServers): @@ -25,7 +26,7 @@ def test_dlck_basic(self): errors = [] dmg = self.get_dmg_command() self.log_step("Create a pool to run dlck") - self.add_pool() + add_pool() self.log_step("Setup dlck parameters to run dlck command") pool_uuids = dmg.get_pool_list_uuids(no_query=True) scm_mount = self.server_managers[0].get_config_value("scm_mount") @@ -46,8 +47,8 @@ def test_dlck_basic(self): result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info(f"dlck basic test output: %s \n", result) + self.log.info("dlck basic test output: %s \n", result) dmg.system_start() if errors: - self.fail(f"Errors detected: %s \n", errors) + self.fail("Errors detected: %s \n", errors) self.log.info("dlck basic test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 29647e013a5..f2b8f237fda 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -10,6 +10,7 @@ from fault_config_utils import FaultInjection from file_utils import distribute_files from run_utils import run_remote +from test_utils_pool import add_pool class DlckBasicFaultTest(TestWithServers): @@ -31,7 +32,7 @@ def test_dlck_basic_fault(self): faults_dict = faults_object.get_faults_dict() fault_list = self.params.get("fault_list", '/run/dlck_test_additional_faults/*') dmg = self.get_dmg_command() - self.add_pool() + add_pool(self) pool_uuids = dmg.get_pool_list_uuids(no_query=True) scm_mount = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: @@ -59,7 +60,7 @@ def test_dlck_basic_fault(self): result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info(f"dlck basic test output: %s \n", result) + self.log.info("dlck basic test output: %s \n", result) # Now, run the other fault injection flags without rebooting or creating any new pools. # Rebooting the servers or creating the new pools will result in injecting fault in # the wrong test code. Fault injections should done only for the dlck alone. @@ -76,7 +77,7 @@ def test_dlck_basic_fault(self): self.log.info("Reading the updated fault injection file contents") with open(fault_inject_file, 'r') as f: file_data = f.read() - self.log.info(f"\n %s", file_data) + self.log.info("\n %s", file_data) distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) if self.server_managers[0].manager.job.using_control_metadata: @@ -87,8 +88,8 @@ def test_dlck_basic_fault(self): env_str=env_str) result = dlck_cmd.run() if not result.passed: - errors.append(f"dlck failed on %s", result.failed_hosts) - self.log.info(f"dlck basic test output: %s", result) + errors.append(f"dlck failed on {result.failed_hosts}") + self.log.info("dlck basic test output: %s", result) dmg.system_start() if not errors: - self.fail("No Errors detected:\n{}".format("\n".join(errors))) + self.fail("No Errors detected:\n %s", errors) From fb11310fc1f70e4acaab5ef317bd96a04d9a8425 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 12 Aug 2026 15:28:04 -0400 Subject: [PATCH 10/78] DAOS-17519 test: Update test timeouts. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 2 +- src/tests/ftest/recovery/dlck_basic_fault.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index 93773b99d1a..99c81bfb620 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -2,7 +2,7 @@ hosts: test_servers: 1 test_clients: 1 -timeout: 500 +timeout: 600 server_config: name: daos_server diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 4c48186841b..20253d207ad 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -2,7 +2,7 @@ hosts: test_servers: 1 test_clients: 1 -timeout: 500 +timeout: 600 server_config: name: daos_server From 7284649f0e5fa2592adf615facb9dc92d5cbdd3a Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 12 Aug 2026 21:55:22 -0400 Subject: [PATCH 11/78] DAOS-17519 test: add_pool needed parameter. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 4 ++-- src/tests/ftest/recovery/dlck_basic_fault.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 18f2578c5ab..341ee7d4432 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -26,7 +26,7 @@ def test_dlck_basic(self): errors = [] dmg = self.get_dmg_command() self.log_step("Create a pool to run dlck") - add_pool() + add_pool(self) self.log_step("Setup dlck parameters to run dlck command") pool_uuids = dmg.get_pool_list_uuids(no_query=True) scm_mount = self.server_managers[0].get_config_value("scm_mount") @@ -50,5 +50,5 @@ def test_dlck_basic(self): self.log.info("dlck basic test output: %s \n", result) dmg.system_start() if errors: - self.fail("Errors detected: %s \n", errors) + self.fail(f"dlck basic test failed with errors: {errors}") self.log.info("dlck basic test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index f2b8f237fda..08365cf4154 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -91,5 +91,6 @@ def test_dlck_basic_fault(self): errors.append(f"dlck failed on {result.failed_hosts}") self.log.info("dlck basic test output: %s", result) dmg.system_start() - if not errors: - self.fail("No Errors detected:\n %s", errors) + if errors: + self.fail(f"dlck basic faulttest failed with errors: {errors}") + self.log.info("dlck basic fault test passed with no errors") \ No newline at end of file From 3b870dce7d84c4f6d26ab8dcf86a5af5a338b5bb Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 12 Aug 2026 22:12:51 -0400 Subject: [PATCH 12/78] DAOS-17519 test: Fix flake8 newline issue. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 08365cf4154..a67f5f8b8bf 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -93,4 +93,4 @@ def test_dlck_basic_fault(self): dmg.system_start() if errors: self.fail(f"dlck basic faulttest failed with errors: {errors}") - self.log.info("dlck basic fault test passed with no errors") \ No newline at end of file + self.log.info("dlck basic fault test passed with no errors") From 68b4f6b4677801da5a92d7461af1df3bc91dd5fc Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 17 Aug 2026 17:32:48 -0400 Subject: [PATCH 13/78] DAOS-17519 test: Python script review updates. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 19 ++++----- src/tests/ftest/recovery/dlck_basic.yaml | 3 -- src/tests/ftest/recovery/dlck_basic_fault.py | 42 +++++++++---------- .../ftest/recovery/dlck_basic_fault.yaml | 6 --- src/tests/ftest/util/dlck_utils.py | 20 +++++---- 5 files changed, 39 insertions(+), 51 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 341ee7d4432..097787d8de8 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -20,34 +20,31 @@ def test_dlck_basic(self): :avocado: tags=all,daily_regression :avocado: tags=hw,medium - :avocado: tags=recovery - :avocado: tags=DlckBasicTest,dlck_cmd,test_dlck_basic + :avocado: tags=recovery,dlck_cmd + :avocado: tags=DlckBasicTest,test_dlck_basic """ errors = [] dmg = self.get_dmg_command() self.log_step("Create a pool to run dlck") - add_pool(self) - self.log_step("Setup dlck parameters to run dlck command") - pool_uuids = dmg.get_pool_list_uuids(no_query=True) + pool = add_pool(self) scm_mount = self.server_managers[0].get_config_value("scm_mount") + host = self.server_managers[0].hosts[0:1] if self.server_managers[0].manager.job.using_control_metadata: log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) control_metadata_dir = os.path.join(log_dir, "control_metadata") daos_control_dir = os.path.join(control_metadata_dir, "daos_control") engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") - dmg.system_stop() - host = self.server_managers[0].hosts[0:1] - if self.server_managers[0].manager.job.using_control_metadata: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, + dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], nvme_conf=nvme_conf, storage_mount=scm_mount) else: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], storage_mount=scm_mount) + self.log_step("Perform dmg system stop to run dlck command") + dmg.system_stop() self.log_step("Run dlck command to check the health of the pool and storage") result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info("dlck basic test output: %s \n", result) dmg.system_start() if errors: self.fail(f"dlck basic test failed with errors: {errors}") diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index 99c81bfb620..dc1137556cd 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -15,8 +15,5 @@ server_config: targets: 4 storage: auto -setup: - start_servers_once: False - pool: size: 50% diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index a67f5f8b8bf..7083109a015 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -23,8 +23,8 @@ def test_dlck_basic_fault(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery - :avocado: tags=DlckBasicFaultTest,dlck_cmd,test_dlck_basic_fault + :avocado: tags=recovery,dlck_cmd + :avocado: tags=DlckBasicFaultTest,test_dlck_basic_fault """ errors = [] faults_dict = {} @@ -32,35 +32,33 @@ def test_dlck_basic_fault(self): faults_dict = faults_object.get_faults_dict() fault_list = self.params.get("fault_list", '/run/dlck_test_additional_faults/*') dmg = self.get_dmg_command() - add_pool(self) - pool_uuids = dmg.get_pool_list_uuids(no_query=True) + pool = add_pool(self) scm_mount = self.server_managers[0].get_config_value("scm_mount") - if self.server_managers[0].manager.job.using_control_metadata: - log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) - control_metadata_dir = os.path.join(log_dir, "control_metadata") - daos_control_dir = os.path.join(control_metadata_dir, "daos_control") - engine_path_dir = os.path.join(daos_control_dir, "engine0") - nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file == "None set for now": self.fail("D_FI_CONFIG environment variable not set") - env_str = "D_FI_CONFIG={} ".format(fault_inject_file) self.log.info("Fault injection file contents") cmd = "cat {}".format(fault_inject_file) host = self.server_managers[0].hosts[0:1] + self.log_step("Run the command to read the fault injection file contents") run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30) - dmg.system_stop() # Run the testing with the first fault which is injected at the beginning of the test. if self.server_managers[0].manager.job.using_control_metadata: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, - storage_mount=scm_mount, env_str=env_str) + log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) + control_metadata_dir = os.path.join(log_dir, "control_metadata") + daos_control_dir = os.path.join(control_metadata_dir, "daos_control") + engine_path_dir = os.path.join(daos_control_dir, "engine0") + nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") + dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], nvme_conf=nvme_conf, + storage_mount=scm_mount) else: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, - env_str=env_str) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], storage_mount=scm_mount) + self.log_step("Perform dmg system stop to run dlck command") + dmg.system_stop() + self.log_step("Run dlck command after injecting the first fault") result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info("dlck basic test output: %s \n", result) # Now, run the other fault injection flags without rebooting or creating any new pools. # Rebooting the servers or creating the new pools will result in injecting fault in # the wrong test code. Fault injections should done only for the dlck alone. @@ -81,15 +79,15 @@ def test_dlck_basic_fault(self): distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) if self.server_managers[0].manager.job.using_control_metadata: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], nvme_conf=nvme_conf, - storage_mount=scm_mount, env_str=env_str) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], nvme_conf=nvme_conf, + storage_mount=scm_mount) else: - dlck_cmd = DlckCommand(host, self.bin, pool_uuids[0], storage_mount=scm_mount, - env_str=env_str) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], storage_mount=scm_mount) + self.log_step("Run dlck command after injecting fault") result = dlck_cmd.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - self.log.info("dlck basic test output: %s", result) + self.log_step("Run the dmg start command") dmg.system_start() if errors: self.fail(f"dlck basic faulttest failed with errors: {errors}") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 20253d207ad..9ef846efc0b 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -12,15 +12,9 @@ server_config: targets: 4 storage: auto -setup: - start_servers_once: True - pool: size: 50% -dlck: - scm_mount: /mnt/daos0 - faults: create_log_dir: fault_list: diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index bf6f820b878..2d81c3a2f40 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -1,18 +1,20 @@ """ - Copyright 2025-2026 Hewlett Packard Enterprise Development LP + Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ +import os -from command_utils_base import CommandWithParameters, FormattedParameter +from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter from run_utils import run_remote -class DlckCommand(CommandWithParameters): +class DlckCommand(ExecutableCommand): """Defines the basic structures of dlck command.""" def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mount=None, - verbose=True, timeout=None, sudo=True, env_str=None): + verbose=True, timeout=None, sudo=True): """Constructor that sets the common variables for sub-commands. Args: @@ -26,13 +28,13 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo timeout (int, optional): Command timeout (sec) used in run. Defaults to None. sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. - env_str (str, optional): Environment variable string to prepend to command. """ super().__init__("/run/dlck/*", "dlck", path) - # Pass environment variable string - self.env_str = "" - if env_str is not None: - self.env_str = str(env_str) + # Pass D_FI_CONFIG environment variable string + fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") + if fault_inject_file == "None set for now": + self.fail("D_FI_CONFIG environment variable not set") + self.env_str = str("D_FI_CONFIG={} ".format(fault_inject_file)) # We need to run with sudo. self.sudo = sudo From a30908f97aaae85db6980b197ee368307e273fd4 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 17 Aug 2026 21:11:23 -0400 Subject: [PATCH 14/78] DAOS-17519 test: Fix the dlck_utils class. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 4 ++-- src/tests/ftest/recovery/dlck_basic_fault.py | 8 ++++---- src/tests/ftest/util/dlck_utils.py | 9 +-------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 097787d8de8..7ed5a78038b 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -35,10 +35,10 @@ def test_dlck_basic(self): daos_control_dir = os.path.join(control_metadata_dir, "daos_control") engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") - dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], nvme_conf=nvme_conf, + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, storage_mount=scm_mount) else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command to check the health of the pool and storage") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 7083109a015..98d7b49deb3 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -49,10 +49,10 @@ def test_dlck_basic_fault(self): daos_control_dir = os.path.join(control_metadata_dir, "daos_control") engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") - dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], nvme_conf=nvme_conf, + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, storage_mount=scm_mount) else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command after injecting the first fault") @@ -79,10 +79,10 @@ def test_dlck_basic_fault(self): distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) if self.server_managers[0].manager.job.using_control_metadata: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], nvme_conf=nvme_conf, + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, storage_mount=scm_mount) else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid[0], storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) self.log_step("Run dlck command after injecting fault") result = dlck_cmd.run() if not result.passed: diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 2d81c3a2f40..c96e07069cf 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -30,11 +30,6 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. """ super().__init__("/run/dlck/*", "dlck", path) - # Pass D_FI_CONFIG environment variable string - fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") - if fault_inject_file == "None set for now": - self.fail("D_FI_CONFIG environment variable not set") - self.env_str = str("D_FI_CONFIG={} ".format(fault_inject_file)) # We need to run with sudo. self.sudo = sudo @@ -64,8 +59,6 @@ def __str__(self): str: the command with all the defined parameters """ value = super().__str__() - if self.sudo: - value = " ".join(["sudo -E -n", value]) return value def run(self): @@ -82,5 +75,5 @@ def run(self): CommandResult: groups of command results from the same hosts with the same return status """ return run_remote( - self.log, self.host, command=self.env_str + str(self), verbose=self.verbose, + self.log, self.host, command=str(self), verbose=self.verbose, timeout=self.timeout) From 60316046f6f645e2df8b4b8b918fbdb609ffe833 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 18 Aug 2026 17:52:41 -0400 Subject: [PATCH 15/78] DAOS-17519 test: Address dlck_basic_fault test failure. Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index c96e07069cf..2f7c87ef584 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -30,9 +30,13 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. """ super().__init__("/run/dlck/*", "dlck", path) - - # We need to run with sudo. - self.sudo = sudo + # Get the fault injection file path and set environment string for the command. + fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") + # Pass environment variable string + self.env_str = "" + self.env_str = "D_FI_CONFIG={} ".format(fault_inject_file) + # We need to run with sudo -E -n + self.dlck_sudo = sudo self.host = server_host @@ -59,6 +63,10 @@ def __str__(self): str: the command with all the defined parameters """ value = super().__str__() + if self.dlck_sudo: + value = " ".join(["sudo -E -n", value]) + return value + return value def run(self): @@ -75,5 +83,5 @@ def run(self): CommandResult: groups of command results from the same hosts with the same return status """ return run_remote( - self.log, self.host, command=str(self), verbose=self.verbose, + self.log, self.host, command=self.env_str + str(self), verbose=self.verbose, timeout=self.timeout) From 1dcadf5d04554b929bda44b75e42e0626022ee84 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 18 Aug 2026 18:31:19 -0400 Subject: [PATCH 16/78] DAOS-17519 test: Implement dlck_utils.py to use ExecutableCommand class. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 31 ++++++++++++------------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 2f7c87ef584..ec580d8cd6d 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -10,7 +10,7 @@ from run_utils import run_remote -class DlckCommand(ExecutableCommand): +class DlckCommand: """Defines the basic structures of dlck command.""" def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mount=None, @@ -29,14 +29,13 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo None. sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. """ - super().__init__("/run/dlck/*", "dlck", path) - # Get the fault injection file path and set environment string for the command. + self.command = ExecutableCommand("/run/dlck/*", "dlck", path) + self.log = self.command.log + + # Add the fault injection file to the environment of the remote command. fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") - # Pass environment variable string - self.env_str = "" - self.env_str = "D_FI_CONFIG={} ".format(fault_inject_file) - # We need to run with sudo -E -n - self.dlck_sudo = sudo + self.command.env["D_FI_CONFIG"] = fault_inject_file + self.command.sudo = sudo self.host = server_host @@ -46,15 +45,15 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo # Pool UUID. (--file pool_uuid[,target_id]) if pool_uuid: - self.pool_uuid = FormattedParameter("--file={}", pool_uuid) + self.command.pool_uuid = FormattedParameter("--file={}", pool_uuid) # NVMe config file path. (--nvme nvme_conf) if nvme_conf: - self.nvme = FormattedParameter("--nvme={}", nvme_conf) + self.command.nvme = FormattedParameter("--nvme={}", nvme_conf) # Storage mount point. (--storage storage_mount) if storage_mount: - self.storage_mount = FormattedParameter("--storage={}", storage_mount) + self.command.storage_mount = FormattedParameter("--storage={}", storage_mount) def __str__(self): """Return the command with all of its defined parameters as a string. @@ -62,12 +61,7 @@ def __str__(self): Returns: str: the command with all the defined parameters """ - value = super().__str__() - if self.dlck_sudo: - value = " ".join(["sudo -E -n", value]) - return value - - return value + return self.command.with_exports def run(self): """Run the dlck command. @@ -83,5 +77,4 @@ def run(self): CommandResult: groups of command results from the same hosts with the same return status """ return run_remote( - self.log, self.host, command=self.env_str + str(self), verbose=self.verbose, - timeout=self.timeout) + self.log, self.host, command=str(self), verbose=self.verbose, timeout=self.timeout) From 391e9263b5a8dc11b20fb5bc9cbfe02214537d12 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 19 Aug 2026 11:00:20 -0400 Subject: [PATCH 17/78] DAOS-17519 test: Fix the error checking issues. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 2 +- src/tests/ftest/util/dlck_utils.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 98d7b49deb3..1792ea8a9af 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -89,6 +89,6 @@ def test_dlck_basic_fault(self): errors.append(f"dlck failed on {result.failed_hosts}") self.log_step("Run the dmg start command") dmg.system_start() - if errors: + if not errors: self.fail(f"dlck basic faulttest failed with errors: {errors}") self.log.info("dlck basic fault test passed with no errors") diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index ec580d8cd6d..d9e796c90b2 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -34,7 +34,8 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo # Add the fault injection file to the environment of the remote command. fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") - self.command.env["D_FI_CONFIG"] = fault_inject_file + if fault_inject_file != "None set for now": + self.command.env["D_FI_CONFIG"] = fault_inject_file self.command.sudo = sudo self.host = server_host From a0406f1ce4fe1b3e179e1240ac1d89f68c3f1316 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 19 Aug 2026 22:14:22 -0400 Subject: [PATCH 18/78] DAOS-17519 test: Run basic test without passing env. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 2 +- src/tests/ftest/recovery/dlck_basic_fault.yaml | 2 +- src/tests/ftest/util/dlck_utils.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index dc1137556cd..e31f85a8e78 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -1,5 +1,5 @@ hosts: - test_servers: 1 + test_servers: 2 test_clients: 1 timeout: 600 diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 9ef846efc0b..2852b651c84 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -1,5 +1,5 @@ hosts: - test_servers: 1 + test_servers: 2 test_clients: 1 timeout: 600 diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index d9e796c90b2..80cd947e1ab 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -32,10 +32,12 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo self.command = ExecutableCommand("/run/dlck/*", "dlck", path) self.log = self.command.log + self.run_sudo_with_exports = False # Add the fault injection file to the environment of the remote command. fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file != "None set for now": self.command.env["D_FI_CONFIG"] = fault_inject_file + self.run_sudo_with_exports = True self.command.sudo = sudo self.host = server_host @@ -62,7 +64,9 @@ def __str__(self): Returns: str: the command with all the defined parameters """ - return self.command.with_exports + if self.run_sudo_with_exports: + return self.command.with_exports + return str(self.command) def run(self): """Run the dlck command. From 9c314c7110edbe952b0cfe28bc3b5c5e78b158e7 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 20 Aug 2026 15:46:30 -0400 Subject: [PATCH 19/78] DAOS-17519 test: Create a separate dlck logger. Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 80cd947e1ab..f1ccaf4d3e9 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -7,6 +7,7 @@ from command_utils import ExecutableCommand from command_utils_base import FormattedParameter +from logging import getLogger from run_utils import run_remote @@ -30,9 +31,8 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. """ self.command = ExecutableCommand("/run/dlck/*", "dlck", path) - self.log = self.command.log + self._logger = getLogger(__name__) - self.run_sudo_with_exports = False # Add the fault injection file to the environment of the remote command. fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file != "None set for now": @@ -64,9 +64,7 @@ def __str__(self): Returns: str: the command with all the defined parameters """ - if self.run_sudo_with_exports: - return self.command.with_exports - return str(self.command) + return self.command.with_exports def run(self): """Run the dlck command. @@ -82,4 +80,4 @@ def run(self): CommandResult: groups of command results from the same hosts with the same return status """ return run_remote( - self.log, self.host, command=str(self), verbose=self.verbose, timeout=self.timeout) + self._logger, self.host, command=str(self), verbose=self.verbose, timeout=self.timeout) From 04898a8cfbb77b455a1d3c60f0b6c8949325308a Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 20 Aug 2026 15:50:01 -0400 Subject: [PATCH 20/78] DAOS-17519 test: Removed unused variable. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index f1ccaf4d3e9..8ddd93a6d75 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -37,7 +37,6 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file != "None set for now": self.command.env["D_FI_CONFIG"] = fault_inject_file - self.run_sudo_with_exports = True self.command.sudo = sudo self.host = server_host From 903d0160fffd41fbc4ccc284547851b6d5792f30 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 20 Aug 2026 17:39:24 -0400 Subject: [PATCH 21/78] DAOS-17519 test: Fix the pylint issue. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 8ddd93a6d75..341e905295f 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -4,10 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os +from logging import getLogger from command_utils import ExecutableCommand from command_utils_base import FormattedParameter -from logging import getLogger from run_utils import run_remote From 7d3f84e1f52e0ee035d0ac484347e92d51b22f55 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 20 Aug 2026 17:42:55 -0400 Subject: [PATCH 22/78] DAOS-17519 test: Update the pool size to 80 percent. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 2 +- src/tests/ftest/recovery/dlck_basic_fault.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index e31f85a8e78..e59614e4202 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -16,4 +16,4 @@ server_config: storage: auto pool: - size: 50% + size: 80% diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 2852b651c84..8fd27367019 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -13,7 +13,7 @@ server_config: storage: auto pool: - size: 50% + size: 80% faults: create_log_dir: From 035fc9534386ac16f074222498510ca2b7fc3915 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Sun, 23 Aug 2026 10:50:53 -0400 Subject: [PATCH 23/78] DAOS-17519 test: Remove the fault injection file after test completion. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 1792ea8a9af..85d1101f37e 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -9,7 +9,7 @@ from dlck_utils import DlckCommand from fault_config_utils import FaultInjection from file_utils import distribute_files -from run_utils import run_remote +from run_utils import command_as_user, run_remote from test_utils_pool import add_pool @@ -89,6 +89,15 @@ def test_dlck_basic_fault(self): errors.append(f"dlck failed on {result.failed_hosts}") self.log_step("Run the dmg start command") dmg.system_start() + # Make sure to delete the fi.yaml file after the test + if os.path.exists(fault_inject_file): + os.remove(fault_inject_file) + # Remove the fault injection file from all remote hosts as well + result = run_remote(self.log, self.hostlist_servers, command_as_user(f'rm -fr {fault_inject_file}', + self.root)) + if not result.passed: + self.log.error(f'Error removing {fault_inject_file} on {result.failed_hosts}') + if not errors: self.fail(f"dlck basic faulttest failed with errors: {errors}") self.log.info("dlck basic fault test passed with no errors") From 47973fc68c80726a4430e667eda6360febafb8e0 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Sun, 23 Aug 2026 10:54:07 -0400 Subject: [PATCH 24/78] DAOS-17519 test: Fix minor lint issue. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 85d1101f37e..203bf163612 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -93,8 +93,9 @@ def test_dlck_basic_fault(self): if os.path.exists(fault_inject_file): os.remove(fault_inject_file) # Remove the fault injection file from all remote hosts as well - result = run_remote(self.log, self.hostlist_servers, command_as_user(f'rm -fr {fault_inject_file}', - self.root)) + result = run_remote(self.log, self.hostlist_servers, + command_as_user(f'rm -fr {fault_inject_file}', + self.root)) if not result.passed: self.log.error(f'Error removing {fault_inject_file} on {result.failed_hosts}') From e0cba861dbcac5a436efa074be2113ff62a63743 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Sun, 23 Aug 2026 13:23:26 -0400 Subject: [PATCH 25/78] DAOS-17519 test: Pass user information as string. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 203bf163612..dacef924ddc 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -95,7 +95,7 @@ def test_dlck_basic_fault(self): # Remove the fault injection file from all remote hosts as well result = run_remote(self.log, self.hostlist_servers, command_as_user(f'rm -fr {fault_inject_file}', - self.root)) + "root")) if not result.passed: self.log.error(f'Error removing {fault_inject_file} on {result.failed_hosts}') From 1e89fc4470c6ad0176ff03f222897d357c83ab58 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 00:16:45 -0400 Subject: [PATCH 26/78] DAOS-17519 test: Remove with_exports property. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 2 +- src/tests/ftest/util/dlck_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index dacef924ddc..50dbb065ebc 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -97,7 +97,7 @@ def test_dlck_basic_fault(self): command_as_user(f'rm -fr {fault_inject_file}', "root")) if not result.passed: - self.log.error(f'Error removing {fault_inject_file} on {result.failed_hosts}') + self.log.info("Error removing %s on %s", fault_inject_file, result.failed_hosts) if not errors: self.fail(f"dlck basic faulttest failed with errors: {errors}") diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 341e905295f..c33d684c03d 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -63,7 +63,7 @@ def __str__(self): Returns: str: the command with all the defined parameters """ - return self.command.with_exports + return str(self.command) def run(self): """Run the dlck command. From 6eea9f36313c1c71c2814373f2aeb02cd839fb8b Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 08:45:34 -0400 Subject: [PATCH 27/78] DAOS-17519 test: Set the system_ram_reserved to 16. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 4 ++-- src/tests/ftest/recovery/dlck_basic_fault.py | 11 +---------- src/tests/ftest/recovery/dlck_basic_fault.yaml | 7 +++++-- src/tests/ftest/util/dlck_utils.py | 2 +- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index e59614e4202..f90ed5fece3 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -3,10 +3,10 @@ hosts: test_clients: 1 timeout: 600 - +system_ram_reserved: 16 server_config: name: daos_server - engines_per_host: 1 + engines_per_host: 2 engines: 0: targets: 4 diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 50dbb065ebc..2fde41c3115 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -9,7 +9,7 @@ from dlck_utils import DlckCommand from fault_config_utils import FaultInjection from file_utils import distribute_files -from run_utils import command_as_user, run_remote +from run_utils import run_remote from test_utils_pool import add_pool @@ -89,15 +89,6 @@ def test_dlck_basic_fault(self): errors.append(f"dlck failed on {result.failed_hosts}") self.log_step("Run the dmg start command") dmg.system_start() - # Make sure to delete the fi.yaml file after the test - if os.path.exists(fault_inject_file): - os.remove(fault_inject_file) - # Remove the fault injection file from all remote hosts as well - result = run_remote(self.log, self.hostlist_servers, - command_as_user(f'rm -fr {fault_inject_file}', - "root")) - if not result.passed: - self.log.info("Error removing %s on %s", fault_inject_file, result.failed_hosts) if not errors: self.fail(f"dlck basic faulttest failed with errors: {errors}") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 8fd27367019..d39c50c114f 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -3,14 +3,17 @@ hosts: test_clients: 1 timeout: 600 - +system_ram_reserved: 16 server_config: name: daos_server - engines_per_host: 1 + engines_per_host: 2 engines: 0: targets: 4 storage: auto + 1: + targets: 4 + storage: auto pool: size: 80% diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index c33d684c03d..7a0358db48e 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -63,7 +63,7 @@ def __str__(self): Returns: str: the command with all the defined parameters """ - return str(self.command) + return self.command.with_exports() def run(self): """Run the dlck command. From e87061930b814321493a7c753efb1dff202e24c6 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 08:52:28 -0400 Subject: [PATCH 28/78] DAOS-17519 test: Add with_exports back to the dlck_utils. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-unit-test: true Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 7a0358db48e..341e905295f 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -63,7 +63,7 @@ def __str__(self): Returns: str: the command with all the defined parameters """ - return self.command.with_exports() + return self.command.with_exports def run(self): """Run the dlck command. From becb00119c977f289ab09dec5e0eb96dd1a2b71a Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 10:57:24 -0400 Subject: [PATCH 29/78] DAOS-17519 test: Add the --log_dir parameter to dlck. Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 7 ++++- src/utils/dlck/cmds/dlck_cmd_check.c | 18 +++++++++++- src/utils/dlck/dlck_args.h | 4 ++- src/utils/dlck/dlck_args_common.c | 6 ++++ src/utils/dlck/tests/dlck_args_ut.c | 43 ++++++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 341e905295f..7094920ca2b 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -15,7 +15,7 @@ class DlckCommand: """Defines the basic structures of dlck command.""" def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mount=None, - verbose=True, timeout=None, sudo=True): + log_dir=None, verbose=True, timeout=None, sudo=True): """Constructor that sets the common variables for sub-commands. Args: @@ -24,6 +24,7 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo pool_uuid (str, optional): Pool UUID. Defaults to None. nvme_conf (str, optional): NVMe config file path. Defaults to None. storage_mount (str, optional): Storage mount point. Defaults to None. + log_dir (str, optional): Log directory. Defaults to None. verbose (bool, optional): Display command output in run. Defaults to True. timeout (int, optional): Command timeout (sec) used in run. Defaults to @@ -57,6 +58,10 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo if storage_mount: self.command.storage_mount = FormattedParameter("--storage={}", storage_mount) + # log dir. (--log_dir log_dir) + if log_dir: + self.command.log_dir = FormattedParameter("--log_dir={}", log_dir) + def __str__(self): """Return the command with all of its defined parameters as a string. diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index b48116b6dd6..d03c0aca1a5 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -8,6 +8,10 @@ #include #include +#include +#include +#include + #include "../dlck_args.h" #include "../dlck_bitmap.h" #include "../dlck_checker.h" @@ -146,11 +150,23 @@ dlck_cmd_check(struct dlck_control *ctrl) D_ASSERT(ctrl != NULL); struct checker *ck = &ctrl->checker; - char log_dir_template[] = "/tmp/dlck_check_XXXXXX"; + const char *log_root = ctrl->common.log_dir; + char log_dir_template[PATH_MAX]; struct dlck_engine *engine = NULL; int *rcs; + int len; int rc; + if (log_root == NULL || log_root[0] == '\0') + log_root = "/tmp"; + + len = snprintf(log_dir_template, sizeof(log_dir_template), "%s/dlck_check_XXXXXX", log_root); + if (len < 0 || len >= sizeof(log_dir_template)) { + rc = -DER_INVAL; + CK_PRINTL_RC(ck, rc, "Invalid log directory path"); + return rc; + } + /** create a log directory */ if (DAOS_FAIL_CHECK(DLCK_FAULT_CREATE_LOG_DIR)) { /** fault injection */ ctrl->log_dir = NULL; diff --git a/src/utils/dlck/dlck_args.h b/src/utils/dlck/dlck_args.h index 4cac4865659..12f374a0bd5 100644 --- a/src/utils/dlck/dlck_args.h +++ b/src/utils/dlck/dlck_args.h @@ -27,6 +27,7 @@ #define KEY_COMMON_OPTIONS 'o' #define KEY_COMMON_WRITE_MODE 'w' #define KEY_COMMON_VERBOSE 'v' +#define KEY_COMMON_LOG_DIR 'l' #define KEY_FILES 'f' /** the options below follow the daos_engine options */ #define KEY_ENGINE_NUMA_NODE 'p' @@ -47,8 +48,9 @@ struct dlck_args_common { struct checker_options options; - bool write_mode; /** false by default (dry run) */ + bool write_mode; /** false by default (dry run) */ bool verbose; /** false by default */ + char *log_dir; /** NULL by default -> /tmp */ }; /** diff --git a/src/utils/dlck/dlck_args_common.c b/src/utils/dlck/dlck_args_common.c index d11a56239c6..eaf98e96ef2 100644 --- a/src/utils/dlck/dlck_args_common.c +++ b/src/utils/dlck/dlck_args_common.c @@ -16,6 +16,8 @@ static struct argp_option args_common_options[] = { OPT_HEADER("Options:", GROUP_OPTIONS), /** entries below inherits the group number of the header entry */ {"write_mode", KEY_COMMON_WRITE_MODE, 0, 0, "Make changes persistent."}, + {"log_dir", KEY_COMMON_LOG_DIR, "PATH", 0, + "Directory where per-run dlck checker logs are created. Defaults to /tmp."}, {"options", KEY_COMMON_OPTIONS, "OPTIONS", 0, "Set options. Options are comma-separated and may include arguments using the equals sign " "('='). Please see available options below."}, @@ -43,6 +45,7 @@ args_common_init(struct dlck_args_common *args) /** set defaults */ args->write_mode = false; /** dry run */ args->verbose = false; + args->log_dir = NULL; args->options.cko_non_zero_padding = CHECKER_EVENT_WARNING; } @@ -95,6 +98,9 @@ args_common_parser(int key, char *arg, struct argp_state *state) case KEY_COMMON_VERBOSE: args->verbose = true; break; + case KEY_COMMON_LOG_DIR: + args->log_dir = arg; + break; case KEY_COMMON_OPTIONS: rc = args_common_options_parse(arg, &args->options, state); break; diff --git a/src/utils/dlck/tests/dlck_args_ut.c b/src/utils/dlck/tests/dlck_args_ut.c index b4df0899ba5..b131e39c259 100644 --- a/src/utils/dlck/tests/dlck_args_ut.c +++ b/src/utils/dlck/tests/dlck_args_ut.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,7 @@ extern struct argp argp_engine; struct dlck_control Ctrl; +argp_parser_t Argp_common_parser_real; argp_parser_t Argp_engine_parser_real; /** wrappers and mocks */ @@ -69,6 +71,22 @@ argp_engine_parser_mock(int key, char *arg, struct argp_state *state) return 0; } +static error_t +argp_file_parser_noop(int key, char *arg, struct argp_state *state) +{ + assert_non_null(state); + assert_ptr_equal(state->input, &Ctrl.files); + return 0; +} + +static error_t +argp_engine_parser_noop(int key, char *arg, struct argp_state *state) +{ + assert_non_null(state); + assert_ptr_equal(state->input, &Ctrl.engine); + return 0; +} + /** setups & teardowns */ static int @@ -128,16 +146,41 @@ test_engine_parser_END_no_storage_path_fail(void **state_ptr) assert_int_equal(ret, PARSER_FAILURE); } +static void +test_parser_log_dir_option(void **unused) +{ + argp_parser_t orig_common = argp_common.parser; + argp_parser_t orig_file = argp_file.parser; + argp_parser_t orig_engine = argp_engine.parser; + int argc = 2; + char *argv[] = {APP_NAME_MOCK, "--log_dir=/var/tmp/dlck_logs"}; + + memset(&Ctrl, 0, sizeof(Ctrl)); + argp_common.parser = Argp_common_parser_real; + argp_file.parser = argp_file_parser_noop; + argp_engine.parser = argp_engine_parser_noop; + + dlck_args_parse(argc, argv, &Ctrl); + assert_non_null(Ctrl.common.log_dir); + assert_string_equal(Ctrl.common.log_dir, "/var/tmp/dlck_logs"); + + argp_common.parser = orig_common; + argp_file.parser = orig_file; + argp_engine.parser = orig_engine; +} + static const struct CMUnitTest dlck_args_tests[] = { {"DLCK_ARGS100: parser - children connection", test_parser_children_connection, NULL, NULL}, {"DLCK_ARGS200: engine parser + ARGP_KEY_END + no storage path", test_engine_parser_END_no_storage_path_fail, setup_engine_args_default, NULL}, + {"DLCK_ARGS300: parser --log_dir option", test_parser_log_dir_option, NULL, NULL}, }; int main(int argc, char **argv) { /** collect function pointers to real parsers */ + Argp_common_parser_real = argp_common.parser; Argp_engine_parser_real = argp_engine.parser; /** overwrite real parsers with mocks */ From 890c9767a0f0bb894148ccd0796922adbe510a68 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 11:01:44 -0400 Subject: [PATCH 30/78] DAOS-17519 test: Pass the log_dir path while calling dlck. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 5 +++-- src/tests/ftest/recovery/dlck_basic_fault.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 7ed5a78038b..1daf8de6f2d 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -36,9 +36,10 @@ def test_dlck_basic(self): engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, - storage_mount=scm_mount) + storage_mount=scm_mount, log_dir=self.log_dir) else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount, + log_dir=self.log_dir) self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command to check the health of the pool and storage") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 2fde41c3115..1e321c331d8 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -50,9 +50,10 @@ def test_dlck_basic_fault(self): engine_path_dir = os.path.join(daos_control_dir, "engine0") nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, - storage_mount=scm_mount) + storage_mount=scm_mount, log_dir=self.log_dir) else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount, + log_dir=self.log_dir ) self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command after injecting the first fault") @@ -80,9 +81,10 @@ def test_dlck_basic_fault(self): fault_inject_file) if self.server_managers[0].manager.job.using_control_metadata: dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, - storage_mount=scm_mount) + storage_mount=scm_mount, log_dir=self.log_dir) else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount, + log_dir=self.log_dir) self.log_step("Run dlck command after injecting fault") result = dlck_cmd.run() if not result.passed: From 3f038d8e5cbad1e60baf05887626156e318084c3 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 11:13:32 -0400 Subject: [PATCH 31/78] DAOS-17519 test: Revert the system_ram_reserved changes. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 1 - src/tests/ftest/recovery/dlck_basic_fault.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index f90ed5fece3..23f743bc686 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -3,7 +3,6 @@ hosts: test_clients: 1 timeout: 600 -system_ram_reserved: 16 server_config: name: daos_server engines_per_host: 2 diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index d39c50c114f..be376d2a95a 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -3,7 +3,6 @@ hosts: test_clients: 1 timeout: 600 -system_ram_reserved: 16 server_config: name: daos_server engines_per_host: 2 From b4dd96a9d17b4a3dd126480af9afe49a1fe2954b Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 11:31:39 -0400 Subject: [PATCH 32/78] DAOS-17519 test: Fix the DAOS build error. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/utils/dlck/cmds/dlck_cmd_check.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index d03c0aca1a5..08c027f56f4 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -8,8 +8,6 @@ #include #include -#include -#include #include #include "../dlck_args.h" @@ -151,32 +149,31 @@ dlck_cmd_check(struct dlck_control *ctrl) struct checker *ck = &ctrl->checker; const char *log_root = ctrl->common.log_dir; - char log_dir_template[PATH_MAX]; + char *log_dir_template = NULL; struct dlck_engine *engine = NULL; int *rcs; - int len; int rc; if (log_root == NULL || log_root[0] == '\0') log_root = "/tmp"; - len = snprintf(log_dir_template, sizeof(log_dir_template), "%s/dlck_check_XXXXXX", log_root); - if (len < 0 || len >= sizeof(log_dir_template)) { - rc = -DER_INVAL; - CK_PRINTL_RC(ck, rc, "Invalid log directory path"); - return rc; - } - /** create a log directory */ if (DAOS_FAIL_CHECK(DLCK_FAULT_CREATE_LOG_DIR)) { /** fault injection */ ctrl->log_dir = NULL; errno = daos_fail_value_get(); } else { + D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", log_root); + if (log_dir_template == NULL) { + rc = -DER_NOMEM; + CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); + return rc; + } ctrl->log_dir = mkdtemp(log_dir_template); } if (ctrl->log_dir == NULL) { rc = daos_errno2der(errno); CK_PRINTL_RC(ck, rc, "Cannot create log directory"); + D_FREE(log_dir_template); return rc; } CK_PRINTF(ck, "Log directory: %s\n", ctrl->log_dir); @@ -185,7 +182,7 @@ dlck_cmd_check(struct dlck_control *ctrl) rc = dlck_engine_start(&ctrl->engine, &engine); CK_APPENDL_RC(ck, rc); if (rc != DER_SUCCESS) { - return rc; + goto err_log_dir_template; } if (d_list_empty(&ctrl->files.list)) { @@ -231,6 +228,7 @@ dlck_cmd_check(struct dlck_control *ctrl) /** Ignore an error for now to print the collected results. */ dlck_report_results(rcs, ctrl->engine.targets, ctrl->warnings_num, ck); D_FREE(rcs); + D_FREE(log_dir_template); /** Return the first encountered error. */ return rc; @@ -239,6 +237,8 @@ dlck_cmd_check(struct dlck_control *ctrl) D_FREE(rcs); err_stop_engine: (void)dlck_engine_stop(engine); +err_log_dir_template: + D_FREE(log_dir_template); return rc; } From 50253bc934c6fd361f78534b01cf8efdd86793e0 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 19:14:50 -0400 Subject: [PATCH 33/78] DAOS-17519 test: Add the logfile to the YAML file. Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 2 ++ src/tests/ftest/recovery/dlck_basic_fault.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index 23f743bc686..e1781f47905 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -9,9 +9,11 @@ server_config: engines: 0: targets: 4 + log_file: daos_server0.log storage: auto 1: targets: 4 + log_file: daos_server1.log storage: auto pool: diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index be376d2a95a..6b00d5a0cba 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -9,9 +9,11 @@ server_config: engines: 0: targets: 4 + log_file: daos_server0.log storage: auto 1: targets: 4 + log_file: daos_server1.log storage: auto pool: From ee085bc63e3c5a1c2189c9df7e1965804037af9c Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 19:16:50 -0400 Subject: [PATCH 34/78] DAOS-17519 test: fix pylint issue. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-test-rpms: true Skip-build-leap15: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 1e321c331d8..224ee32f542 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -53,7 +53,7 @@ def test_dlck_basic_fault(self): storage_mount=scm_mount, log_dir=self.log_dir) else: dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount, - log_dir=self.log_dir ) + log_dir=self.log_dir) self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command after injecting the first fault") From 7fd7d82abcb43c11ffad42af9497083bc0fc2af0 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 21:55:48 -0400 Subject: [PATCH 35/78] DAOS-17519 test: Adding --verbose to dlck command. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 7094920ca2b..d498bc7e739 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -46,6 +46,9 @@ def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mo self.verbose = verbose self.timeout = timeout + if self.verbose: + self.command.dlck_verbose = FormattedParameter("--verbose", verbose) + # Pool UUID. (--file pool_uuid[,target_id]) if pool_uuid: self.command.pool_uuid = FormattedParameter("--file={}", pool_uuid) From 5af0ef158273056a15f24d0289cbed983a0696c2 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 24 Aug 2026 22:01:17 -0400 Subject: [PATCH 36/78] DAOS-17519 test: Fix C lint issues. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/utils/dlck/dlck_args_common.c | 4 ++-- src/utils/dlck/tests/dlck_args_ut.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/dlck/dlck_args_common.c b/src/utils/dlck/dlck_args_common.c index eaf98e96ef2..764de97e4c0 100644 --- a/src/utils/dlck/dlck_args_common.c +++ b/src/utils/dlck/dlck_args_common.c @@ -16,8 +16,8 @@ static struct argp_option args_common_options[] = { OPT_HEADER("Options:", GROUP_OPTIONS), /** entries below inherits the group number of the header entry */ {"write_mode", KEY_COMMON_WRITE_MODE, 0, 0, "Make changes persistent."}, - {"log_dir", KEY_COMMON_LOG_DIR, "PATH", 0, - "Directory where per-run dlck checker logs are created. Defaults to /tmp."}, + {"log_dir", KEY_COMMON_LOG_DIR, "PATH", 0, + "Directory where per-run dlck checker logs are created. Defaults to /tmp."}, {"options", KEY_COMMON_OPTIONS, "OPTIONS", 0, "Set options. Options are comma-separated and may include arguments using the equals sign " "('='). Please see available options below."}, diff --git a/src/utils/dlck/tests/dlck_args_ut.c b/src/utils/dlck/tests/dlck_args_ut.c index b131e39c259..57a5289fca8 100644 --- a/src/utils/dlck/tests/dlck_args_ut.c +++ b/src/utils/dlck/tests/dlck_args_ut.c @@ -173,7 +173,7 @@ static const struct CMUnitTest dlck_args_tests[] = { {"DLCK_ARGS100: parser - children connection", test_parser_children_connection, NULL, NULL}, {"DLCK_ARGS200: engine parser + ARGP_KEY_END + no storage path", test_engine_parser_END_no_storage_path_fail, setup_engine_args_default, NULL}, - {"DLCK_ARGS300: parser --log_dir option", test_parser_log_dir_option, NULL, NULL}, + {"DLCK_ARGS300: parser --log_dir option", test_parser_log_dir_option, NULL, NULL}, }; int From 067c4d1781f13631b173e016f0f9119c2a3de72c Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 25 Aug 2026 15:19:45 -0400 Subject: [PATCH 37/78] DAOS-17519 test: Modified the files based on all the code review updates. Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 24 +-- src/tests/ftest/recovery/dlck_basic_fault.py | 35 ++-- src/tests/ftest/util/command_utils.py | 172 ++++++++++++++++++- src/tests/ftest/util/dlck_utils.py | 107 ++++-------- src/tests/ftest/util/fault_config_utils.py | 22 ++- src/utils/dlck/cmds/dlck_cmd_check.c | 15 +- src/utils/dlck/dlck_args.h | 7 +- src/utils/dlck/dlck_args_common.c | 4 +- src/utils/dlck/tests/dlck_args_ut.c | 43 ----- 9 files changed, 252 insertions(+), 177 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 1daf8de6f2d..c80499f0d70 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -6,11 +6,11 @@ import os from apricot import TestWithServers -from dlck_utils import DlckCommand +from dlck_utils import TestDlck from test_utils_pool import add_pool -class DlckBasicTest(TestWithServers): +class DlckBasicTest(TestDlck): """Test class for dlck command line utility. :avocado: recursive @@ -27,23 +27,17 @@ def test_dlck_basic(self): dmg = self.get_dmg_command() self.log_step("Create a pool to run dlck") pool = add_pool(self) - scm_mount = self.server_managers[0].get_config_value("scm_mount") - host = self.server_managers[0].hosts[0:1] + dlck = self.get_dlck_command() + dlck.pool_uuid.value = pool.uuid + dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: - log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) - control_metadata_dir = os.path.join(log_dir, "control_metadata") - daos_control_dir = os.path.join(control_metadata_dir, "daos_control") - engine_path_dir = os.path.join(daos_control_dir, "engine0") - nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, - storage_mount=scm_mount, log_dir=self.log_dir) - else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount, - log_dir=self.log_dir) + dlck.nvme.value = os.path.join( + self.server_managers[0].get_config_value("path"), "daos_control", "engine0", + "daos_nvme.conf") self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command to check the health of the pool and storage") - result = dlck_cmd.run() + result = dlck.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") dmg.system_start() diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 224ee32f542..f6e9fa901ac 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -5,15 +5,14 @@ """ import os -from apricot import TestWithServers -from dlck_utils import DlckCommand +from dlck_utils import TestDlck from fault_config_utils import FaultInjection from file_utils import distribute_files from run_utils import run_remote from test_utils_pool import add_pool -class DlckBasicFaultTest(TestWithServers): +class DlckBasicFaultTest(TestDlck): """Test class for dlck command line utility. :avocado: recursive @@ -33,31 +32,25 @@ def test_dlck_basic_fault(self): fault_list = self.params.get("fault_list", '/run/dlck_test_additional_faults/*') dmg = self.get_dmg_command() pool = add_pool(self) - scm_mount = self.server_managers[0].get_config_value("scm_mount") + dlck = self.get_dlck_command() + dlck.pool_uuid.value = pool.uuid + dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file == "None set for now": self.fail("D_FI_CONFIG environment variable not set") self.log.info("Fault injection file contents") cmd = "cat {}".format(fault_inject_file) - host = self.server_managers[0].hosts[0:1] self.log_step("Run the command to read the fault injection file contents") run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30) # Run the testing with the first fault which is injected at the beginning of the test. if self.server_managers[0].manager.job.using_control_metadata: - log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) - control_metadata_dir = os.path.join(log_dir, "control_metadata") - daos_control_dir = os.path.join(control_metadata_dir, "daos_control") - engine_path_dir = os.path.join(daos_control_dir, "engine0") - nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, - storage_mount=scm_mount, log_dir=self.log_dir) - else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount, - log_dir=self.log_dir) + dlck.nvme.value = os.path.join( + self.server_managers[0].get_config_value("path"), "daos_control", "engine0", + "daos_nvme.conf") self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command after injecting the first fault") - result = dlck_cmd.run() + result = dlck.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") # Now, run the other fault injection flags without rebooting or creating any new pools. @@ -79,19 +72,13 @@ def test_dlck_basic_fault(self): self.log.info("\n %s", file_data) distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) - if self.server_managers[0].manager.job.using_control_metadata: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, - storage_mount=scm_mount, log_dir=self.log_dir) - else: - dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount, - log_dir=self.log_dir) self.log_step("Run dlck command after injecting fault") - result = dlck_cmd.run() + result = dlck.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") self.log_step("Run the dmg start command") dmg.system_start() if not errors: - self.fail(f"dlck basic faulttest failed with errors: {errors}") + self.fail(f"dlck basic fault test failed with errors: {errors}") self.log.info("dlck basic fault test passed with no errors") diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index f5d2885f94e..e4d128ecca7 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -23,7 +23,7 @@ from file_utils import change_file_owner, create_directory, distribute_files from general_utils import (DaosTestError, check_file_exists, get_file_listing, get_job_manager_class, get_subprocess_stdout, run_command) -from run_utils import command_as_user, run_remote +from run_utils import command_as_user, run_local, run_remote, stop_processes from user_utils import get_primary_group from yaml_utils import get_yaml_data @@ -95,6 +95,9 @@ def __init__(self, namespace, command, path="", subprocess=False, check_results= if check_results: self.check_results_list = list(check_results) + # Internal flag used to indicate if cleanup is required for the command. + self.__cleanup_needed = False + def __str__(self): """Return the command with all of its defined parameters as a string. @@ -206,6 +209,7 @@ def run(self, raise_exception=None): CommandFailure: if there is an error running the command """ + self.__cleanup_needed = True if self.run_as_subprocess: self._run_subprocess() return None @@ -263,7 +267,7 @@ def check_results(self): if self.result and self.check_results_list: regex = r"({})".format("|".join(self.check_results_list)) self.log.debug("Checking the command output for any bad keywords: %s", regex) - for output in (self.result.stdout_text, self.result.stderr_text): + for output in (self._result_stdout(), self._result_stderr()): match = re.findall(regex, output) if match: self.log.info( @@ -275,6 +279,26 @@ def check_results(self): break return status + def _result_stdout(self): + """Get all the stdout from the command result. + + Returns: + str: the command result's stdout as a string + """ + if not self.result: + raise CommandFailure("No command result available to return stdout") + return self.result.stdout_text + + def _result_stderr(self): + """Get all the stderr from the command result. + + Returns: + str: the command result's stderr as a string + """ + if not self.result: + raise CommandFailure("No command result available to return stderr") + return self.result.stderr_text + def _run_subprocess(self): """Run the command as a sub process. @@ -372,6 +396,35 @@ def stop(self): self.log.info("%s stopped successfully", self.command) self._process = None + def cleanup_command(self): + """Cleanup the command.""" + if not self.__cleanup_needed: + self.log.info("No cleanup needed for %s", self.command) + return + + self.log.info("Cleaning up %s", self.command) + regex = self.command_regex + if self.full_command_regex: + regex = f"'{str(self)}'" + hosts = None + if hasattr(self, "hosts"): + hosts = self.hosts + detected, running = stop_processes( + self.log, hosts, regex, full_command=self.full_command_regex) + if not detected: + self.log.info( + "No remote %s processes killed on %s (none found), done.", + regex, "local host" if not hosts else hosts) + elif running: + self.log.info( + "***Unable to kill remote %s process on %s! Please investigate/report.***", + regex, running) + else: + self.log.info( + "***At least one remote %s process needed to be killed on %s! Please investigate/" + "report.***", regex, detected) + self.__cleanup_needed = False + def wait(self): """Wait for the sub process to complete. @@ -447,7 +500,7 @@ def get_output(self, method_name, regex_method=None, **kwargs): # Parse the output and return if not regex_method: regex_method = method_name - return self.parse_output(result.stdout_text, regex_method) + return self.parse_output(self._result_stdout(), regex_method) def parse_output(self, stdout, regex_method): """Parse output using findall() with supplied 'regex_method' as pattern. @@ -464,7 +517,7 @@ def parse_output(self, stdout, regex_method): """ if regex_method not in self.METHOD_REGEX: - raise CommandFailure("No pattern regex defined for '{}()'".format(regex_method)) + raise CommandFailure(f"No pattern regex defined for '{regex_method}()'") return re.findall(self.METHOD_REGEX[regex_method], stdout) def get_params(self, test): @@ -1511,3 +1564,112 @@ def __init__(self, run_user='root'): self.unit_command = BasicParameter(None, position=1) self.service = BasicParameter(None, position=2) + + +class RunCommand(ExecutableCommand): + """A class for command run with run_remote()/run_local().""" + + def __init__(self, namespace, command, path="", check_results=None, run_user=None): + """Create a RunCommand object. + + Uses run_remote()/run_local() to run a command str provided. + + Args: + namespace (str): yaml namespace (path to parameters) + command (str): string of the command to be executed. + path (str, optional): path to location of command binary file. Defaults to "". + check_results (list, optional): list of words used to mark the command as failed if + any are found in the command output. Defaults to None. + run_user (str, optional): user to run as. Defaults to None, which will run commands as + the current user. + """ + super().__init__(namespace, command, path, False, check_results, run_user) + self._hosts = None + + @property + def hosts(self): + """Get the host(s) on which to remotely run the command via run(). + + Returns: + NodeSet: remote host(s) on which the command will run. + """ + return self._hosts + + @hosts.setter + def hosts(self, value): + """Set the host(s) on which to remotely run the command via run(). + + If the specified host is None the command will run locally w/o ssh. + + Args: + value (NodeSet): remote host(s) on which to run the command + + Raises: + TypeError: if value is not a NodeSet + """ + if not isinstance(value, NodeSet): + raise TypeError(f"Invalid {self.command} host NodeSet: {value} ({type(value)})") + self._hosts = value.copy() + + def _run_process(self, raise_exception=None): + """Run the command as a foreground process. + + Args: + raise_exception (bool, optional): whether or not to raise an exception if the command + fails. This overrides the self.exit_status_exception + setting if defined. Defaults to None. + + Raises: + CommandFailure: if there is an error running the command with raise_exception or + self.exit_status_exception (when raise_exception is None) set to True. + + Returns: + CommandResult: result from running the command + """ + if raise_exception is None: + raise_exception = self.exit_status_exception + + self.result = None + if not self.hosts: + result = run_local(self.log, self.with_exports, self.verbose, self.timeout) + else: + result = run_remote(self.log, self.hosts, self.with_exports, self.verbose, self.timeout) + self.result = result + + if raise_exception and not result.passed: + raise CommandFailure(f"Error running {self.command} on: {result.failed_hosts}") + + if raise_exception and not self.check_results(): + raise CommandFailure( + f"Error running {self.command}: a {self.check_results_list} keyword was detected") + + return result + + def _result_stdout(self): + """Get all the stdout from the command result. + + Returns: + str: the command result's stdout as a string + """ + if not self.result: + raise CommandFailure("No command result available to return stdout") + return self.result.joined_stdout + + def _result_stderr(self): + """Get all the stderr from the command result. + + Returns: + str: the command result's stderr as a string + """ + if not self.result: + raise CommandFailure("No command result available to return stderr") + return self.result.joined_stderr + + def _get_new(self): + """Get a new object based upon this one. + + Returns: + RunCommand: a new RunCommand object + """ + return RunCommand( + self.namespace, self._command, self._path, self.check_results_list, self.run_user) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index d498bc7e739..c905c3037ef 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -4,87 +4,54 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -from logging import getLogger -from command_utils import ExecutableCommand +from apricot import TestWithServers +from command_utils import RunCommand from command_utils_base import FormattedParameter -from run_utils import run_remote -class DlckCommand: - """Defines the basic structures of dlck command.""" +class TestDlck(TestWithServers): + # pylint: disable=too-few-public-methods + """Base class for Dlck tests. - def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mount=None, - log_dir=None, verbose=True, timeout=None, sudo=True): - """Constructor that sets the common variables for sub-commands. + :avocado: recursive + """ - Args: - server_host (NodeSet): Server host to run the command. - path (str): path to the dlck command. - pool_uuid (str, optional): Pool UUID. Defaults to None. - nvme_conf (str, optional): NVMe config file path. Defaults to None. - storage_mount (str, optional): Storage mount point. Defaults to None. - log_dir (str, optional): Log directory. Defaults to None. - verbose (bool, optional): Display command output in run. - Defaults to True. - timeout (int, optional): Command timeout (sec) used in run. Defaults to - None. - sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. - """ - self.command = ExecutableCommand("/run/dlck/*", "dlck", path) - self._logger = getLogger(__name__) - - # Add the fault injection file to the environment of the remote command. - fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") - if fault_inject_file != "None set for now": - self.command.env["D_FI_CONFIG"] = fault_inject_file - self.command.sudo = sudo + def get_dlck_command(self, path="", namespace="/run/dlck/*"): + """Get a DlckCommand object with parameters from the test yaml file. - self.host = server_host - - # Members needed for run(). - self.verbose = verbose - self.timeout = timeout - - if self.verbose: - self.command.dlck_verbose = FormattedParameter("--verbose", verbose) - - # Pool UUID. (--file pool_uuid[,target_id]) - if pool_uuid: - self.command.pool_uuid = FormattedParameter("--file={}", pool_uuid) - - # NVMe config file path. (--nvme nvme_conf) - if nvme_conf: - self.command.nvme = FormattedParameter("--nvme={}", nvme_conf) + Args: + path (str, optional): path to location of command binary file. Defaults to "". + namespace (str, optional): path to yaml parameters. Defaults to "/run/dlck/*". - # Storage mount point. (--storage storage_mount) - if storage_mount: - self.command.storage_mount = FormattedParameter("--storage={}", storage_mount) + Returns: + DlckCommand: a DlckCommand object with parameters from the test yaml file + """ + dlck = DlckCommand(path, namespace) + self.register_cleanup(dlck.cleanup_command) + dlck.hosts = self.server_managers[0].hosts[0:1] + dlck.log_file = self.log_dir + dlck.get_params(self) + return dlck - # log dir. (--log_dir log_dir) - if log_dir: - self.command.log_dir = FormattedParameter("--log_dir={}", log_dir) - def __str__(self): - """Return the command with all of its defined parameters as a string. +class DlckCommand(RunCommand): + """Defines a object representing a dlck command.""" - Returns: - str: the command with all the defined parameters - """ - return self.command.with_exports + def __init__(self, path="", namespace="/run/dlck/*"): + """Create a DlckCommand object. - def run(self): - """Run the dlck command. Args: - host (NodeSet): Host(s) on which to run the command. - command (str): Environment Variable string + dlck sub-command to run. - verbose (bool, optional): Display command output in run. - Defaults to True. - timeout (int, optional): Command timeout (sec) used in run. Defaults to - None. - - Returns: - CommandResult: groups of command results from the same hosts with the same return status + path (str, optional): path to location of command binary file. Defaults to "". + namespace (str, optional): path to yaml parameters. Defaults to "/run/dlck/*". """ - return run_remote( - self._logger, self.host, command=str(self), verbose=self.verbose, timeout=self.timeout) + super().__init__(namespace, "dlck", path) + self.pool_uuid = FormattedParameter("--file={}", None) + self.nvme = FormattedParameter("--nvme={}", None) + self.storage_mount = FormattedParameter("--storage={}", None) + self.log_dir = FormattedParameter("--log_dir={}", None) + + # Add the fault injection file to the environment. + fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") + if fault_inject_file != "None set for now": + self.env["D_FI_CONFIG"] = fault_inject_file diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index 4721ec691ac..9043cac477d 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -278,17 +278,29 @@ 'DAOS_FAULT_POOL_OPEN_UUID': { 'id': '131589', 'interval': '2'}, # skip sys_db - 'DAOS_FAULT_BTREE_OPEN_INV_CLASS': { + 'DAOS_FAULT_BTREE_OPEN_INV_CLASS_28': { 'id': '131590', 'interval': '28', # containers tree fine-tuned 'max_faults': '1'}, - 'DAOS_FAULT_BTREE_OPEN_UNREG_CLASS': { + 'DAOS_FAULT_BTREE_OPEN_INV_CLASS_29': { + 'id': '131590', + 'interval': '29', # gc tree fine-tuned + 'max_faults': '1'}, + 'DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_28': { + 'id': '131591', + 'interval': '28', # containers tree fine-tuned + 'max_faults': '1'}, + 'DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_29': { 'id': '131591', - 'interval': '29', + 'interval': '29', # gc tree fine-tuned + 'max_faults': '1'}, + 'DAOS_FAULT_BTREE_FEATURES_28': { + 'id': '131592', + 'interval': '28', # containers tree fine-tuned 'max_faults': '1'}, - 'DAOS_FAULT_BTREE_FEATURES': { + 'DAOS_FAULT_BTREE_FEATURES_29': { 'id': '131592', - 'interval': '28', + 'interval': '29', # gc tree fine-tuned 'max_faults': '1'}, 'DAOS_FAULT_POOL_EXT_PADDING': { 'id': '131593', diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index 08c027f56f4..c6e8ad0d1a9 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -148,21 +148,17 @@ dlck_cmd_check(struct dlck_control *ctrl) D_ASSERT(ctrl != NULL); struct checker *ck = &ctrl->checker; - const char *log_root = ctrl->common.log_dir; char *log_dir_template = NULL; struct dlck_engine *engine = NULL; int *rcs; int rc; - if (log_root == NULL || log_root[0] == '\0') - log_root = "/tmp"; - /** create a log directory */ if (DAOS_FAIL_CHECK(DLCK_FAULT_CREATE_LOG_DIR)) { /** fault injection */ ctrl->log_dir = NULL; errno = daos_fail_value_get(); } else { - D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", log_root); + D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir) if (log_dir_template == NULL) { rc = -DER_NOMEM; CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); @@ -173,8 +169,7 @@ dlck_cmd_check(struct dlck_control *ctrl) if (ctrl->log_dir == NULL) { rc = daos_errno2der(errno); CK_PRINTL_RC(ck, rc, "Cannot create log directory"); - D_FREE(log_dir_template); - return rc; + goto err_free_template; } CK_PRINTF(ck, "Log directory: %s\n", ctrl->log_dir); @@ -182,7 +177,7 @@ dlck_cmd_check(struct dlck_control *ctrl) rc = dlck_engine_start(&ctrl->engine, &engine); CK_APPENDL_RC(ck, rc); if (rc != DER_SUCCESS) { - goto err_log_dir_template; + goto err_free_template; } if (d_list_empty(&ctrl->files.list)) { @@ -237,7 +232,7 @@ dlck_cmd_check(struct dlck_control *ctrl) D_FREE(rcs); err_stop_engine: (void)dlck_engine_stop(engine); -err_log_dir_template: +err_free_template: D_FREE(log_dir_template); return rc; diff --git a/src/utils/dlck/dlck_args.h b/src/utils/dlck/dlck_args.h index 12f374a0bd5..7ec2ff11380 100644 --- a/src/utils/dlck/dlck_args.h +++ b/src/utils/dlck/dlck_args.h @@ -41,6 +41,7 @@ #define DLCK_DEFAULT_MAX_DMA_BUF_SIZE 5120 #define DLCK_DEFAULT_NVME_HUGEPAGE_SIZE 2 #define DLCK_DEFAULT_TARGETS 4 +#define DLCK_DEFAULT_LOG_DIR "/tmp" #define DLCK_TARGET_MAX 31 @@ -48,9 +49,9 @@ struct dlck_args_common { struct checker_options options; - bool write_mode; /** false by default (dry run) */ - bool verbose; /** false by default */ - char *log_dir; /** NULL by default -> /tmp */ + bool write_mode; + bool verbose; + char *log_dir; }; /** diff --git a/src/utils/dlck/dlck_args_common.c b/src/utils/dlck/dlck_args_common.c index 764de97e4c0..83b3f8a9745 100644 --- a/src/utils/dlck/dlck_args_common.c +++ b/src/utils/dlck/dlck_args_common.c @@ -17,7 +17,7 @@ static struct argp_option args_common_options[] = { /** entries below inherits the group number of the header entry */ {"write_mode", KEY_COMMON_WRITE_MODE, 0, 0, "Make changes persistent."}, {"log_dir", KEY_COMMON_LOG_DIR, "PATH", 0, - "Directory where per-run dlck checker logs are created. Defaults to /tmp."}, + "Log directory. Default: " DLCK_DEFAULT_LOG_DIR "."}, {"options", KEY_COMMON_OPTIONS, "OPTIONS", 0, "Set options. Options are comma-separated and may include arguments using the equals sign " "('='). Please see available options below."}, @@ -45,7 +45,7 @@ args_common_init(struct dlck_args_common *args) /** set defaults */ args->write_mode = false; /** dry run */ args->verbose = false; - args->log_dir = NULL; + args->log_dir = DLCK_DEFAULT_LOG_DIR; args->options.cko_non_zero_padding = CHECKER_EVENT_WARNING; } diff --git a/src/utils/dlck/tests/dlck_args_ut.c b/src/utils/dlck/tests/dlck_args_ut.c index 57a5289fca8..b4df0899ba5 100644 --- a/src/utils/dlck/tests/dlck_args_ut.c +++ b/src/utils/dlck/tests/dlck_args_ut.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -27,7 +26,6 @@ extern struct argp argp_engine; struct dlck_control Ctrl; -argp_parser_t Argp_common_parser_real; argp_parser_t Argp_engine_parser_real; /** wrappers and mocks */ @@ -71,22 +69,6 @@ argp_engine_parser_mock(int key, char *arg, struct argp_state *state) return 0; } -static error_t -argp_file_parser_noop(int key, char *arg, struct argp_state *state) -{ - assert_non_null(state); - assert_ptr_equal(state->input, &Ctrl.files); - return 0; -} - -static error_t -argp_engine_parser_noop(int key, char *arg, struct argp_state *state) -{ - assert_non_null(state); - assert_ptr_equal(state->input, &Ctrl.engine); - return 0; -} - /** setups & teardowns */ static int @@ -146,41 +128,16 @@ test_engine_parser_END_no_storage_path_fail(void **state_ptr) assert_int_equal(ret, PARSER_FAILURE); } -static void -test_parser_log_dir_option(void **unused) -{ - argp_parser_t orig_common = argp_common.parser; - argp_parser_t orig_file = argp_file.parser; - argp_parser_t orig_engine = argp_engine.parser; - int argc = 2; - char *argv[] = {APP_NAME_MOCK, "--log_dir=/var/tmp/dlck_logs"}; - - memset(&Ctrl, 0, sizeof(Ctrl)); - argp_common.parser = Argp_common_parser_real; - argp_file.parser = argp_file_parser_noop; - argp_engine.parser = argp_engine_parser_noop; - - dlck_args_parse(argc, argv, &Ctrl); - assert_non_null(Ctrl.common.log_dir); - assert_string_equal(Ctrl.common.log_dir, "/var/tmp/dlck_logs"); - - argp_common.parser = orig_common; - argp_file.parser = orig_file; - argp_engine.parser = orig_engine; -} - static const struct CMUnitTest dlck_args_tests[] = { {"DLCK_ARGS100: parser - children connection", test_parser_children_connection, NULL, NULL}, {"DLCK_ARGS200: engine parser + ARGP_KEY_END + no storage path", test_engine_parser_END_no_storage_path_fail, setup_engine_args_default, NULL}, - {"DLCK_ARGS300: parser --log_dir option", test_parser_log_dir_option, NULL, NULL}, }; int main(int argc, char **argv) { /** collect function pointers to real parsers */ - Argp_common_parser_real = argp_common.parser; Argp_engine_parser_real = argp_engine.parser; /** overwrite real parsers with mocks */ From 0b0c7837f66920477b3f0c138fde5aad68829e22 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 25 Aug 2026 15:21:53 -0400 Subject: [PATCH 38/78] DAOS-17519 test: Update the Copyright information. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/utils/dlck/dlck_args.h | 2 +- src/utils/dlck/dlck_args_common.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/dlck/dlck_args.h b/src/utils/dlck/dlck_args.h index 7ec2ff11380..1af24ffbb0a 100644 --- a/src/utils/dlck/dlck_args.h +++ b/src/utils/dlck/dlck_args.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ diff --git a/src/utils/dlck/dlck_args_common.c b/src/utils/dlck/dlck_args_common.c index 83b3f8a9745..1a48bf81511 100644 --- a/src/utils/dlck/dlck_args_common.c +++ b/src/utils/dlck/dlck_args_common.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ From edf4949b2ccadcb7fb62e6bbb0636a90fea02502 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 25 Aug 2026 15:24:20 -0400 Subject: [PATCH 39/78] DAOS-17519 test: Reduce the pool size to 10 percent. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 9 ++------- src/tests/ftest/recovery/dlck_basic_fault.yaml | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index e1781f47905..38ff0cedc5c 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -5,16 +5,11 @@ hosts: timeout: 600 server_config: name: daos_server - engines_per_host: 2 + engines_per_host: 1 engines: 0: targets: 4 - log_file: daos_server0.log - storage: auto - 1: - targets: 4 - log_file: daos_server1.log storage: auto pool: - size: 80% + size: 10% diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 6b00d5a0cba..7712e06c24b 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -5,19 +5,14 @@ hosts: timeout: 600 server_config: name: daos_server - engines_per_host: 2 + engines_per_host: 1 engines: 0: targets: 4 - log_file: daos_server0.log - storage: auto - 1: - targets: 4 - log_file: daos_server1.log storage: auto pool: - size: 80% + size: 10% faults: create_log_dir: From 5178e62f846f0b4fc923aacc19ab5a2f7b21dbe1 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 25 Aug 2026 16:43:32 -0400 Subject: [PATCH 40/78] DAOS-17519 test: Updated new faults and allow exception for dlck fault testing. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- .../ftest/recovery/dlck_basic_fault.yaml | 14 ++++++-- src/tests/ftest/util/dlck_utils.py | 1 + src/tests/ftest/util/fault_config_utils.py | 32 +++++++++++++++---- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 7712e06c24b..214bb4c80b7 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -21,6 +21,11 @@ faults: dlck_test_additional_faults: fault_list: + - DLCK_MOCK_ROOT + - DLCK_FAULT_GETGRNAM + - DLCK_MOCK_NO_DAOS_SERVER_GROUP + - DLCK_FAULT_GETGROUPS + - DLCK_MOCK_NOT_IN_DAOS_SERVER_GROUP - DLCK_FAULT_CREATE_POOL_DIR - DLCK_FAULT_ENGINE_START - DLCK_FAULT_ENGINE_EXEC @@ -32,8 +37,11 @@ dlck_test_additional_faults: - DAOS_FAULT_POOL_OPEN_MAGIC - DAOS_FAULT_POOL_OPEN_VERSION - DAOS_FAULT_POOL_OPEN_UUID - - DAOS_FAULT_BTREE_OPEN_INV_CLASS - - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS - - DAOS_FAULT_BTREE_FEATURES + - DAOS_FAULT_BTREE_OPEN_INV_CLASS_28 + - DAOS_FAULT_BTREE_OPEN_INV_CLASS_29 + - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_28 + - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_29 + - DAOS_FAULT_BTREE_FEATURES_28 + - DAOS_FAULT_BTREE_FEATURES_29 - DAOS_FAULT_POOL_EXT_PADDING - DAOS_FAULT_POOL_EXT_RESERVED diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index c905c3037ef..6ca17dbd82a 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -55,3 +55,4 @@ def __init__(self, path="", namespace="/run/dlck/*"): fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file != "None set for now": self.env["D_FI_CONFIG"] = fault_inject_file + self.exit_status_exception = False diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index 9043cac477d..c8ed924ecd0 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -236,30 +236,50 @@ 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, - 'DLCK_FAULT_CREATE_LOG_DIR': { + 'DLCK_MOCK_ROOT': { 'id': '131328', 'interval': '1', 'max_faults': '1'}, - 'DLCK_FAULT_CREATE_POOL_DIR': { + 'DLCK_FAULT_GETGRNAM': { 'id': '131329', 'interval': '1', 'max_faults': '1'}, - 'DLCK_FAULT_ENGINE_START': { + 'DLCK_MOCK_NO_DAOS_SERVER_GROUP': { 'id': '131330', 'interval': '1', 'max_faults': '1'}, - 'DLCK_FAULT_ENGINE_EXEC': { + 'DLCK_FAULT_GETGROUPS': { 'id': '131331', 'interval': '1', 'max_faults': '1'}, - 'DLCK_FAULT_ENGINE_JOIN': { + 'DLCK_MOCK_NOT_IN_DAOS_SERVER_GROUP': { 'id': '131332', 'interval': '1', 'max_faults': '1'}, - 'DLCK_FAULT_ENGINE_STOP': { + 'DLCK_FAULT_CREATE_LOG_DIR': { 'id': '131333', 'interval': '1', 'max_faults': '1'}, + 'DLCK_FAULT_CREATE_POOL_DIR': { + 'id': '131334', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_START': { + 'id': '131335', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_EXEC': { + 'id': '131336', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_JOIN': { + 'id': '131337', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_STOP': { + 'id': '131338', + 'interval': '1', + 'max_faults': '1'}, 'DAOS_FAULT_POOL_NVME_HEALTH': { 'id': '131584', 'interval': '2', # skip sys_db From 09e6bd9ca41a877471497d769ea375d249612c91 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 25 Aug 2026 17:04:51 -0400 Subject: [PATCH 41/78] DAOS-17519 test: Address the scons dlck compile error. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/utils/dlck/cmds/dlck_cmd_check.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index c6e8ad0d1a9..ea33cd8cc54 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -158,7 +158,7 @@ dlck_cmd_check(struct dlck_control *ctrl) ctrl->log_dir = NULL; errno = daos_fail_value_get(); } else { - D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir) + D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir); if (log_dir_template == NULL) { rc = -DER_NOMEM; CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); From 0a6b0beddc2e9c2eceb87d2d11ffcde86c2721ed Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 26 Aug 2026 21:08:58 -0400 Subject: [PATCH 42/78] DAOS-17519 test: More review updates and check errors. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 5 ++- src/tests/ftest/recovery/dlck_basic_fault.py | 39 ++++++++++++++------ src/tests/ftest/util/command_utils.py | 15 +++++++- src/tests/ftest/util/dlck_utils.py | 2 +- src/tests/ftest/util/fault_config_utils.py | 2 +- src/utils/dlck/cmds/dlck_cmd_check.c | 13 ++++--- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index c80499f0d70..46c8505e7f7 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -5,7 +5,6 @@ """ import os -from apricot import TestWithServers from dlck_utils import TestDlck from test_utils_pool import add_pool @@ -29,6 +28,7 @@ def test_dlck_basic(self): pool = add_pool(self) dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid + dlck.log_dir = self.test_env.log_dir dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: dlck.nvme.value = os.path.join( @@ -37,7 +37,8 @@ def test_dlck_basic(self): self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command to check the health of the pool and storage") - result = dlck.run() + with dlck.no_exception(): + result = dlck.run() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") dmg.system_start() diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index f6e9fa901ac..48e5bc0b8ea 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os +import re from dlck_utils import TestDlck from fault_config_utils import FaultInjection @@ -17,6 +18,18 @@ class DlckBasicFaultTest(TestDlck): :avocado: recursive """ + def check_dlck_result(self, result, fault_name): + """Check that dlck reports the expected error for an injected fault.""" + expected_errors = ("DER_INVAL", "DER_NONEXIST", "DER_ID_MISMATCH", + "DER_DF_INCOMPT", "DER_DF_INVAL", "WARNING:") + warning_pattern = r"\b\d+\s+warning(?:\(s\)|s?)\b" + output = f"{result.joined_stdout}\n{result.joined_stderr}" + expected_error_found = any(error in output for error in expected_errors) + warning_found = re.search(warning_pattern, output) + if not (expected_error_found or warning_found): + return f"{fault_name}: no expected error found in dlck output: {output}" + return None + def test_dlck_basic_fault(self): """Basic Fault Test: Run 'dlck' injecting basic faults. @@ -26,7 +39,6 @@ def test_dlck_basic_fault(self): :avocado: tags=DlckBasicFaultTest,test_dlck_basic_fault """ errors = [] - faults_dict = {} faults_object = FaultInjection() faults_dict = faults_object.get_faults_dict() fault_list = self.params.get("fault_list", '/run/dlck_test_additional_faults/*') @@ -34,12 +46,13 @@ def test_dlck_basic_fault(self): pool = add_pool(self) dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid + dlck.log_dir = self.test_env.log_dir dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") if fault_inject_file == "None set for now": self.fail("D_FI_CONFIG environment variable not set") self.log.info("Fault injection file contents") - cmd = "cat {}".format(fault_inject_file) + cmd = f"cat {fault_inject_file}" self.log_step("Run the command to read the fault injection file contents") run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30) # Run the testing with the first fault which is injected at the beginning of the test. @@ -50,12 +63,14 @@ def test_dlck_basic_fault(self): self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command after injecting the first fault") - result = dlck.run() - if not result.passed: - errors.append(f"dlck failed on {result.failed_hosts}") + with dlck.no_exception(): + result = dlck.run() + error = self.check_dlck_result(result, "Initial fault") + if error: + errors.append(error) # Now, run the other fault injection flags without rebooting or creating any new pools. # Rebooting the servers or creating the new pools will result in injecting fault in - # the wrong test code. Fault injections should done only for the dlck alone. + # the wrong test code. Fault injections should be done only for the dlck alone. for test_fault in fault_list: with open(fault_inject_file, 'w') as f: f.write("fault_config:\n") @@ -73,12 +88,14 @@ def test_dlck_basic_fault(self): distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) self.log_step("Run dlck command after injecting fault") - result = dlck.run() - if not result.passed: - errors.append(f"dlck failed on {result.failed_hosts}") + with dlck.no_exception(): + result = dlck.run() + error = self.check_dlck_result(result, test_fault) + if error: + errors.append(error) self.log_step("Run the dmg start command") dmg.system_start() - if not errors: - self.fail(f"dlck basic fault test failed with errors: {errors}") + if errors: + self.fail(f"dlck basic test failed with errors: {errors}") self.log.info("dlck basic fault test passed with no errors") diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index e4d128ecca7..2b0ed39bc5e 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -397,10 +397,14 @@ def stop(self): self._process = None def cleanup_command(self): - """Cleanup the command.""" + """Cleanup the command. + + Returns: + list: a list of errors encountered during cleanup. + """ if not self.__cleanup_needed: self.log.info("No cleanup needed for %s", self.command) - return + return [] self.log.info("Cleaning up %s", self.command) regex = self.command_regex @@ -424,6 +428,7 @@ def cleanup_command(self): "***At least one remote %s process needed to be killed on %s! Please investigate/" "report.***", regex, detected) self.__cleanup_needed = False + return [] def wait(self): """Wait for the sub process to complete. @@ -1585,6 +1590,7 @@ def __init__(self, namespace, command, path="", check_results=None, run_user=Non """ super().__init__(namespace, command, path, False, check_results, run_user) self._hosts = None + self.register_cleanup_method = None @property def hosts(self): @@ -1629,6 +1635,11 @@ def _run_process(self, raise_exception=None): if raise_exception is None: raise_exception = self.exit_status_exception + if callable(self.register_cleanup_method): + # Stop any running processes started by this job manager when the test completes + # pylint: disable=not-callable + self.register_cleanup_method(self.cleanup_command) + self.result = None if not self.hosts: result = run_local(self.log, self.with_exports, self.verbose, self.timeout) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 6ca17dbd82a..018a3c600b7 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -30,7 +30,7 @@ def get_dlck_command(self, path="", namespace="/run/dlck/*"): dlck = DlckCommand(path, namespace) self.register_cleanup(dlck.cleanup_command) dlck.hosts = self.server_managers[0].hosts[0:1] - dlck.log_file = self.log_dir + dlck.log_dir = self.log_dir dlck.get_params(self) return dlck diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index c8ed924ecd0..794ccb8e9b8 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -312,7 +312,7 @@ 'max_faults': '1'}, 'DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_29': { 'id': '131591', - 'interval': '29', # gc tree fine-tuned + 'interval': '29', # gc tree fine-tuned 'max_faults': '1'}, 'DAOS_FAULT_BTREE_FEATURES_28': { 'id': '131592', diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index ea33cd8cc54..a2f9bed2f1d 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -158,14 +158,15 @@ dlck_cmd_check(struct dlck_control *ctrl) ctrl->log_dir = NULL; errno = daos_fail_value_get(); } else { - D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir); - if (log_dir_template == NULL) { - rc = -DER_NOMEM; - CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); - return rc; - } ctrl->log_dir = mkdtemp(log_dir_template); } + /** generate the log directory path template */ + D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir); + if (log_dir_template == NULL) { + rc = -DER_NOMEM; + CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); + return rc; + } if (ctrl->log_dir == NULL) { rc = daos_errno2der(errno); CK_PRINTL_RC(ck, rc, "Cannot create log directory"); From e8a43098f93490b89171004924c77956b36eca28 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 26 Aug 2026 21:19:33 -0400 Subject: [PATCH 43/78] DAOS-17519 test: Remove unwanted code. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 6 ++---- src/tests/ftest/recovery/dlck_basic_fault.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 46c8505e7f7..a67248aa15b 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -37,11 +37,9 @@ def test_dlck_basic(self): self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command to check the health of the pool and storage") - with dlck.no_exception(): - result = dlck.run() + result = dlck.run() + dmg.system_start() if not result.passed: errors.append(f"dlck failed on {result.failed_hosts}") - dmg.system_start() - if errors: self.fail(f"dlck basic test failed with errors: {errors}") self.log.info("dlck basic test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 48e5bc0b8ea..e1438f0be97 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -63,8 +63,7 @@ def test_dlck_basic_fault(self): self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() self.log_step("Run dlck command after injecting the first fault") - with dlck.no_exception(): - result = dlck.run() + result = dlck.run() error = self.check_dlck_result(result, "Initial fault") if error: errors.append(error) @@ -88,8 +87,7 @@ def test_dlck_basic_fault(self): distribute_files(self.log, self.hostlist_servers, fault_inject_file, fault_inject_file) self.log_step("Run dlck command after injecting fault") - with dlck.no_exception(): - result = dlck.run() + result = dlck.run() error = self.check_dlck_result(result, test_fault) if error: errors.append(error) From f32bea4a8ccfe021208d81e1dc87dd6b8aae52da Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 26 Aug 2026 21:25:05 -0400 Subject: [PATCH 44/78] DAOS-17519 test: Address the dlck build error. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/utils/dlck/cmds/dlck_cmd_check.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index a2f9bed2f1d..3387655bf82 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -157,8 +157,6 @@ dlck_cmd_check(struct dlck_control *ctrl) if (DAOS_FAIL_CHECK(DLCK_FAULT_CREATE_LOG_DIR)) { /** fault injection */ ctrl->log_dir = NULL; errno = daos_fail_value_get(); - } else { - ctrl->log_dir = mkdtemp(log_dir_template); } /** generate the log directory path template */ D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir); @@ -167,6 +165,7 @@ dlck_cmd_check(struct dlck_control *ctrl) CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); return rc; } + ctrl->log_dir = mkdtemp(log_dir_template); if (ctrl->log_dir == NULL) { rc = daos_errno2der(errno); CK_PRINTL_RC(ck, rc, "Cannot create log directory"); From 82a724d6e0c2345f31ae66f9aebd9662294a1f56 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 26 Aug 2026 21:27:06 -0400 Subject: [PATCH 45/78] DAOS-17519 test: Minor space issue. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/utils/dlck/cmds/dlck_cmd_check.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index 3387655bf82..d914ce57c3a 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -159,12 +159,12 @@ dlck_cmd_check(struct dlck_control *ctrl) errno = daos_fail_value_get(); } /** generate the log directory path template */ - D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir); - if (log_dir_template == NULL) { - rc = -DER_NOMEM; - CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); - return rc; - } + D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir); + if (log_dir_template == NULL) { + rc = -DER_NOMEM; + CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); + return rc; + } ctrl->log_dir = mkdtemp(log_dir_template); if (ctrl->log_dir == NULL) { rc = daos_errno2der(errno); From 3d4781fdf032cc74f3f9f7daaf369e095b722965 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 26 Aug 2026 21:34:52 -0400 Subject: [PATCH 46/78] DAOS-17519 test: Fix clang errors. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/utils/dlck/cmds/dlck_cmd_check.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index d914ce57c3a..e3793de079e 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -169,7 +169,7 @@ dlck_cmd_check(struct dlck_control *ctrl) if (ctrl->log_dir == NULL) { rc = daos_errno2der(errno); CK_PRINTL_RC(ck, rc, "Cannot create log directory"); - goto err_free_template; + goto err_free_template; } CK_PRINTF(ck, "Log directory: %s\n", ctrl->log_dir); From 33590aad78fb14983526f97f49f897c675004242 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 26 Aug 2026 21:41:23 -0400 Subject: [PATCH 47/78] DAOS-17519 test: More clang errors. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/utils/dlck/dlck_args_common.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/dlck/dlck_args_common.c b/src/utils/dlck/dlck_args_common.c index 1a48bf81511..4169b2f0d2c 100644 --- a/src/utils/dlck/dlck_args_common.c +++ b/src/utils/dlck/dlck_args_common.c @@ -16,8 +16,7 @@ static struct argp_option args_common_options[] = { OPT_HEADER("Options:", GROUP_OPTIONS), /** entries below inherits the group number of the header entry */ {"write_mode", KEY_COMMON_WRITE_MODE, 0, 0, "Make changes persistent."}, - {"log_dir", KEY_COMMON_LOG_DIR, "PATH", 0, - "Log directory. Default: " DLCK_DEFAULT_LOG_DIR "."}, + {"log_dir", KEY_COMMON_LOG_DIR, "PATH", 0, "Log directory. Default: " DLCK_DEFAULT_LOG_DIR}, {"options", KEY_COMMON_OPTIONS, "OPTIONS", 0, "Set options. Options are comma-separated and may include arguments using the equals sign " "('='). Please see available options below."}, From e061e566fd432c4b348f71a1e213af9a40402e34 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 27 Aug 2026 09:23:43 -0400 Subject: [PATCH 48/78] DAOS-17519 test: Reduce the test_servers. Review updates. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 2 +- src/tests/ftest/recovery/dlck_basic_fault.yaml | 2 +- src/utils/dlck/cmds/dlck_cmd_check.c | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index 38ff0cedc5c..125eee7e5c3 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -1,5 +1,5 @@ hosts: - test_servers: 2 + test_servers: 1 test_clients: 1 timeout: 600 diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 214bb4c80b7..237ae559f6d 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -1,5 +1,5 @@ hosts: - test_servers: 2 + test_servers: 1 test_clients: 1 timeout: 600 diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index e3793de079e..124727cf2b7 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -153,11 +153,6 @@ dlck_cmd_check(struct dlck_control *ctrl) int *rcs; int rc; - /** create a log directory */ - if (DAOS_FAIL_CHECK(DLCK_FAULT_CREATE_LOG_DIR)) { /** fault injection */ - ctrl->log_dir = NULL; - errno = daos_fail_value_get(); - } /** generate the log directory path template */ D_ASPRINTF(log_dir_template, "%s/dlck_check_XXXXXX", ctrl->common.log_dir); if (log_dir_template == NULL) { @@ -165,7 +160,13 @@ dlck_cmd_check(struct dlck_control *ctrl) CK_PRINTL_RC(ck, rc, "Cannot allocate log directory path"); return rc; } - ctrl->log_dir = mkdtemp(log_dir_template); + /** create a log directory */ + if (DAOS_FAIL_CHECK(DLCK_FAULT_CREATE_LOG_DIR)) { /** fault injection */ + ctrl->log_dir = NULL; + errno = daos_fail_value_get(); + } else { + ctrl->log_dir = mkdtemp(log_dir_template); + } if (ctrl->log_dir == NULL) { rc = daos_errno2der(errno); CK_PRINTL_RC(ck, rc, "Cannot create log directory"); From 767de9a6e59519e4c34db8e2827cac38d512ff6a Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 28 Aug 2026 13:15:08 -0400 Subject: [PATCH 49/78] DAOS-17519 test: Modify fault injection mechanism for dlck utility. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 25 ++++++++----------- .../ftest/recovery/dlck_basic_fault.yaml | 10 ++------ src/tests/ftest/util/dlck_utils.py | 6 ----- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index e1438f0be97..28761682b54 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -9,7 +9,6 @@ from dlck_utils import TestDlck from fault_config_utils import FaultInjection from file_utils import distribute_files -from run_utils import run_remote from test_utils_pool import add_pool @@ -35,26 +34,27 @@ def test_dlck_basic_fault(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,dlck_cmd + :avocado: tags=recovery,dlck_cmd,faults :avocado: tags=DlckBasicFaultTest,test_dlck_basic_fault """ errors = [] faults_object = FaultInjection() faults_dict = faults_object.get_faults_dict() - fault_list = self.params.get("fault_list", '/run/dlck_test_additional_faults/*') + fault_list = self.params.get("fault_list", '/run/dlck_test_faults/*') + self.log.info("Test log dir %s", self.test_env.log_dir) + # /var/tmp/daos_testing/test_dlck_basic_fault/fi.yaml + fault_inject_file = os.path.join(self.test_env.log_dir, "fi.yaml") + self.log.info("Fault injection file: %s", fault_inject_file) + self.log.info("Faults: %s", fault_list) + self.log.info("Faults dict: %s", faults_dict) dmg = self.get_dmg_command() pool = add_pool(self) dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid dlck.log_dir = self.test_env.log_dir + dlck.exit_status_exception = False + dlck.env["D_FI_CONFIG"] = fault_inject_file dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") - fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") - if fault_inject_file == "None set for now": - self.fail("D_FI_CONFIG environment variable not set") - self.log.info("Fault injection file contents") - cmd = f"cat {fault_inject_file}" - self.log_step("Run the command to read the fault injection file contents") - run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30) # Run the testing with the first fault which is injected at the beginning of the test. if self.server_managers[0].manager.job.using_control_metadata: dlck.nvme.value = os.path.join( @@ -62,11 +62,6 @@ def test_dlck_basic_fault(self): "daos_nvme.conf") self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() - self.log_step("Run dlck command after injecting the first fault") - result = dlck.run() - error = self.check_dlck_result(result, "Initial fault") - if error: - errors.append(error) # Now, run the other fault injection flags without rebooting or creating any new pools. # Rebooting the servers or creating the new pools will result in injecting fault in # the wrong test code. Fault injections should be done only for the dlck alone. diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml index 237ae559f6d..7ecc67f8ae5 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -14,21 +14,15 @@ server_config: pool: size: 10% -faults: - create_log_dir: - fault_list: - - DLCK_FAULT_CREATE_LOG_DIR - -dlck_test_additional_faults: +dlck_test_faults: fault_list: - DLCK_MOCK_ROOT - DLCK_FAULT_GETGRNAM - DLCK_MOCK_NO_DAOS_SERVER_GROUP - - DLCK_FAULT_GETGROUPS - DLCK_MOCK_NOT_IN_DAOS_SERVER_GROUP - DLCK_FAULT_CREATE_POOL_DIR + - DLCK_FAULT_CREATE_LOG_DIR - DLCK_FAULT_ENGINE_START - - DLCK_FAULT_ENGINE_EXEC - DLCK_FAULT_ENGINE_JOIN - DLCK_FAULT_ENGINE_STOP - DAOS_FAULT_POOL_NVME_HEALTH diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 018a3c600b7..4095cbcbc02 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -50,9 +50,3 @@ def __init__(self, path="", namespace="/run/dlck/*"): self.nvme = FormattedParameter("--nvme={}", None) self.storage_mount = FormattedParameter("--storage={}", None) self.log_dir = FormattedParameter("--log_dir={}", None) - - # Add the fault injection file to the environment. - fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") - if fault_inject_file != "None set for now": - self.env["D_FI_CONFIG"] = fault_inject_file - self.exit_status_exception = False From 0891a3cfc95eb0f662c4bd39dd5fb3d2d31c9d04 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 28 Aug 2026 13:24:42 -0400 Subject: [PATCH 50/78] DAOS-17519 test: Remove the unwanted comment. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic_fault.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py index 28761682b54..40109e47fef 100644 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -42,7 +42,6 @@ def test_dlck_basic_fault(self): faults_dict = faults_object.get_faults_dict() fault_list = self.params.get("fault_list", '/run/dlck_test_faults/*') self.log.info("Test log dir %s", self.test_env.log_dir) - # /var/tmp/daos_testing/test_dlck_basic_fault/fi.yaml fault_inject_file = os.path.join(self.test_env.log_dir, "fi.yaml") self.log.info("Fault injection file: %s", fault_inject_file) self.log.info("Faults: %s", fault_list) From db479c69a01cdb0ca97177fce1315fd342327dd4 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 28 Aug 2026 13:43:27 -0400 Subject: [PATCH 51/78] DAOS-17519 test: Fix pylint issue. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 4095cbcbc02..992ee50c336 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -3,8 +3,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import os - from apricot import TestWithServers from command_utils import RunCommand from command_utils_base import FormattedParameter From ee205a071ed55c4d3a07aec13770f2c4037b9108 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 28 Aug 2026 19:49:43 -0400 Subject: [PATCH 52/78] DAOS-17519 test: Merged both the test cases to a single python file. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 79 +++++++++++++++- src/tests/ftest/recovery/dlck_basic.yaml | 28 +++++- src/tests/ftest/recovery/dlck_basic_fault.py | 93 ------------------- .../ftest/recovery/dlck_basic_fault.yaml | 41 -------- src/tests/ftest/util/dlck_utils.py | 2 +- 5 files changed, 103 insertions(+), 140 deletions(-) delete mode 100644 src/tests/ftest/recovery/dlck_basic_fault.py delete mode 100644 src/tests/ftest/recovery/dlck_basic_fault.yaml diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index a67248aa15b..c761f856834 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -4,9 +4,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os +import re from dlck_utils import TestDlck from test_utils_pool import add_pool +from fault_config_utils import FaultInjection +from file_utils import distribute_files class DlckBasicTest(TestDlck): @@ -14,6 +17,18 @@ class DlckBasicTest(TestDlck): :avocado: recursive """ + def check_dlck_result(self, result, fault_name): + """Check that dlck reports the expected error for an injected fault.""" + expected_errors = ("DER_INVAL", "DER_NONEXIST", "DER_ID_MISMATCH", + "DER_DF_INCOMPT", "DER_DF_INVAL", "WARNING:") + warning_pattern = r"\b\d+\s+warning(?:\(s\)|s?)\b" + output = f"{result.joined_stdout}\n{result.joined_stderr}" + expected_error_found = any(error in output for error in expected_errors) + warning_found = re.search(warning_pattern, output) + if not (expected_error_found or warning_found): + return f"{fault_name}: no expected error found in dlck output: {output}" + return None + def test_dlck_basic(self): """Basic Test: Run 'dlck' command @@ -25,11 +40,11 @@ def test_dlck_basic(self): errors = [] dmg = self.get_dmg_command() self.log_step("Create a pool to run dlck") - pool = add_pool(self) + pool = self.get_pool(self) dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid dlck.log_dir = self.test_env.log_dir - dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") + dlck.storage = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: dlck.nvme.value = os.path.join( self.server_managers[0].get_config_value("path"), "daos_control", "engine0", @@ -40,6 +55,62 @@ def test_dlck_basic(self): result = dlck.run() dmg.system_start() if not result.passed: - errors.append(f"dlck failed on {result.failed_hosts}") - self.fail(f"dlck basic test failed with errors: {errors}") + self.fail(f"dlck failed on {result.failed_hosts}") self.log.info("dlck basic test passed with no errors") + + def test_dlck_basic_fault(self): + """Basic Fault Test: Run 'dlck' injecting basic faults. + + :avocado: tags=all,full_regression + :avocado: tags=hw,medium + :avocado: tags=recovery,dlck_cmd,faults + :avocado: tags=DlckBasicTest,test_dlck_basic_fault + """ + errors = [] + faults_object = FaultInjection() + faults_dict = faults_object.get_faults_dict() + fault_list = self.params.get("fault_list", '/run/dlck_test_faults/*') + self.log.info("Test log dir %s", self.test_env.log_dir) + fault_inject_file = os.path.join(self.test_env.log_dir, "fi.yaml") + self.log.info("Fault injection file: %s", fault_inject_file) + self.log.info("Faults: %s", fault_list) + self.log.info("Faults dict: %s", faults_dict) + dmg = self.get_dmg_command() + pool = self.get_pool(self) + dlck = self.get_dlck_command() + dlck.pool_uuid.value = pool.uuid + dlck.log_dir = self.test_env.log_dir + dlck.exit_status_exception = False + dlck.env["D_FI_CONFIG"] = fault_inject_file + dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") + if self.server_managers[0].manager.job.using_control_metadata: + dlck.nvme.value = os.path.join( + self.server_managers[0].get_config_value("path"), "daos_control", "engine0", + "daos_nvme.conf") + self.log_step("Perform dmg system stop to run dlck command") + dmg.system_stop() + for test_fault in fault_list: + with open(fault_inject_file, 'w') as f: + f.write("fault_config:\n") + for count, (key, value) in enumerate(faults_dict[test_fault].items()): + if count == 0: + f.write(f"- {key}: \'{value}\'\n") + else: + f.write(f" {key}: \'{value}\'\n") + self.log.info("Reading the updated fault injection file contents") + with open(fault_inject_file, 'r') as f: + file_data = f.read() + self.log.info("\n %s", file_data) + distribute_files(self.log, self.hostlist_servers, fault_inject_file, + fault_inject_file) + self.log_step("Run dlck command after injecting fault") + result = dlck.run() + error = self.check_dlck_result(result, test_fault) + if error: + errors.append(error) + self.log_step("Run the dmg start command") + dmg.system_start() + + if errors: + self.fail(f"dlck basic test failed with errors: {errors}") + self.log.info("dlck basic fault test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index 125eee7e5c3..ab4b5b25471 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -2,7 +2,7 @@ hosts: test_servers: 1 test_clients: 1 -timeout: 600 +timeout: 420 server_config: name: daos_server engines_per_host: 1 @@ -13,3 +13,29 @@ server_config: pool: size: 10% + +dlck_test_faults: + fault_list: + - DLCK_MOCK_ROOT + - DLCK_FAULT_GETGRNAM + - DLCK_MOCK_NO_DAOS_SERVER_GROUP + - DLCK_MOCK_NOT_IN_DAOS_SERVER_GROUP + - DLCK_FAULT_CREATE_POOL_DIR + - DLCK_FAULT_CREATE_LOG_DIR + - DLCK_FAULT_ENGINE_START + - DLCK_FAULT_ENGINE_JOIN + - DLCK_FAULT_ENGINE_STOP + - DAOS_FAULT_POOL_NVME_HEALTH + - DAOS_FAULT_POOL_OPEN_BIO + - DAOS_FAULT_POOL_OPEN_UMEM + - DAOS_FAULT_POOL_OPEN_MAGIC + - DAOS_FAULT_POOL_OPEN_VERSION + - DAOS_FAULT_POOL_OPEN_UUID + - DAOS_FAULT_BTREE_OPEN_INV_CLASS_28 + - DAOS_FAULT_BTREE_OPEN_INV_CLASS_29 + - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_28 + - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_29 + - DAOS_FAULT_BTREE_FEATURES_28 + - DAOS_FAULT_BTREE_FEATURES_29 + - DAOS_FAULT_POOL_EXT_PADDING + - DAOS_FAULT_POOL_EXT_RESERVED diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py deleted file mode 100644 index 40109e47fef..00000000000 --- a/src/tests/ftest/recovery/dlck_basic_fault.py +++ /dev/null @@ -1,93 +0,0 @@ -""" - Copyright 2026 Hewlett Packard Enterprise Development LP - - SPDX-License-Identifier: BSD-2-Clause-Patent -""" -import os -import re - -from dlck_utils import TestDlck -from fault_config_utils import FaultInjection -from file_utils import distribute_files -from test_utils_pool import add_pool - - -class DlckBasicFaultTest(TestDlck): - """Test class for dlck command line utility. - - :avocado: recursive - """ - def check_dlck_result(self, result, fault_name): - """Check that dlck reports the expected error for an injected fault.""" - expected_errors = ("DER_INVAL", "DER_NONEXIST", "DER_ID_MISMATCH", - "DER_DF_INCOMPT", "DER_DF_INVAL", "WARNING:") - warning_pattern = r"\b\d+\s+warning(?:\(s\)|s?)\b" - output = f"{result.joined_stdout}\n{result.joined_stderr}" - expected_error_found = any(error in output for error in expected_errors) - warning_found = re.search(warning_pattern, output) - if not (expected_error_found or warning_found): - return f"{fault_name}: no expected error found in dlck output: {output}" - return None - - def test_dlck_basic_fault(self): - """Basic Fault Test: Run 'dlck' injecting basic faults. - - :avocado: tags=all,full_regression - :avocado: tags=hw,medium - :avocado: tags=recovery,dlck_cmd,faults - :avocado: tags=DlckBasicFaultTest,test_dlck_basic_fault - """ - errors = [] - faults_object = FaultInjection() - faults_dict = faults_object.get_faults_dict() - fault_list = self.params.get("fault_list", '/run/dlck_test_faults/*') - self.log.info("Test log dir %s", self.test_env.log_dir) - fault_inject_file = os.path.join(self.test_env.log_dir, "fi.yaml") - self.log.info("Fault injection file: %s", fault_inject_file) - self.log.info("Faults: %s", fault_list) - self.log.info("Faults dict: %s", faults_dict) - dmg = self.get_dmg_command() - pool = add_pool(self) - dlck = self.get_dlck_command() - dlck.pool_uuid.value = pool.uuid - dlck.log_dir = self.test_env.log_dir - dlck.exit_status_exception = False - dlck.env["D_FI_CONFIG"] = fault_inject_file - dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") - # Run the testing with the first fault which is injected at the beginning of the test. - if self.server_managers[0].manager.job.using_control_metadata: - dlck.nvme.value = os.path.join( - self.server_managers[0].get_config_value("path"), "daos_control", "engine0", - "daos_nvme.conf") - self.log_step("Perform dmg system stop to run dlck command") - dmg.system_stop() - # Now, run the other fault injection flags without rebooting or creating any new pools. - # Rebooting the servers or creating the new pools will result in injecting fault in - # the wrong test code. Fault injections should be done only for the dlck alone. - for test_fault in fault_list: - with open(fault_inject_file, 'w') as f: - f.write("fault_config:\n") - count = 0 - for key, value in faults_dict[test_fault].items(): - if count == 0: - f.write(f"- {key}: \'{value}\'\n") - else: - f.write(f" {key}: \'{value}\'\n") - count += 1 - self.log.info("Reading the updated fault injection file contents") - with open(fault_inject_file, 'r') as f: - file_data = f.read() - self.log.info("\n %s", file_data) - distribute_files(self.log, self.hostlist_servers, fault_inject_file, - fault_inject_file) - self.log_step("Run dlck command after injecting fault") - result = dlck.run() - error = self.check_dlck_result(result, test_fault) - if error: - errors.append(error) - self.log_step("Run the dmg start command") - dmg.system_start() - - if errors: - self.fail(f"dlck basic test failed with errors: {errors}") - self.log.info("dlck basic fault test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml deleted file mode 100644 index 7ecc67f8ae5..00000000000 --- a/src/tests/ftest/recovery/dlck_basic_fault.yaml +++ /dev/null @@ -1,41 +0,0 @@ -hosts: - test_servers: 1 - test_clients: 1 - -timeout: 600 -server_config: - name: daos_server - engines_per_host: 1 - engines: - 0: - targets: 4 - storage: auto - -pool: - size: 10% - -dlck_test_faults: - fault_list: - - DLCK_MOCK_ROOT - - DLCK_FAULT_GETGRNAM - - DLCK_MOCK_NO_DAOS_SERVER_GROUP - - DLCK_MOCK_NOT_IN_DAOS_SERVER_GROUP - - DLCK_FAULT_CREATE_POOL_DIR - - DLCK_FAULT_CREATE_LOG_DIR - - DLCK_FAULT_ENGINE_START - - DLCK_FAULT_ENGINE_JOIN - - DLCK_FAULT_ENGINE_STOP - - DAOS_FAULT_POOL_NVME_HEALTH - - DAOS_FAULT_POOL_OPEN_BIO - - DAOS_FAULT_POOL_OPEN_UMEM - - DAOS_FAULT_POOL_OPEN_MAGIC - - DAOS_FAULT_POOL_OPEN_VERSION - - DAOS_FAULT_POOL_OPEN_UUID - - DAOS_FAULT_BTREE_OPEN_INV_CLASS_28 - - DAOS_FAULT_BTREE_OPEN_INV_CLASS_29 - - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_28 - - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_29 - - DAOS_FAULT_BTREE_FEATURES_28 - - DAOS_FAULT_BTREE_FEATURES_29 - - DAOS_FAULT_POOL_EXT_PADDING - - DAOS_FAULT_POOL_EXT_RESERVED diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 992ee50c336..00f32508b46 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -46,5 +46,5 @@ def __init__(self, path="", namespace="/run/dlck/*"): super().__init__(namespace, "dlck", path) self.pool_uuid = FormattedParameter("--file={}", None) self.nvme = FormattedParameter("--nvme={}", None) - self.storage_mount = FormattedParameter("--storage={}", None) + self.storage = FormattedParameter("--storage={}", None) self.log_dir = FormattedParameter("--log_dir={}", None) From 9c8b3479d8f1da88f70b3e2a85f25dc998abc8ca Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 28 Aug 2026 19:52:40 -0400 Subject: [PATCH 53/78] DAOS-17519 test: Missed to remove add_pool import. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index c761f856834..1aff386fe33 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -7,7 +7,6 @@ import re from dlck_utils import TestDlck -from test_utils_pool import add_pool from fault_config_utils import FaultInjection from file_utils import distribute_files From 9d199de78c653cbde21add426bf264734d20f913 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 28 Aug 2026 19:54:38 -0400 Subject: [PATCH 54/78] DAOS-17519 test: Removed unused variable. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 1aff386fe33..99164d32eae 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -36,7 +36,6 @@ def test_dlck_basic(self): :avocado: tags=recovery,dlck_cmd :avocado: tags=DlckBasicTest,test_dlck_basic """ - errors = [] dmg = self.get_dmg_command() self.log_step("Create a pool to run dlck") pool = self.get_pool(self) From f78c858bf951f94c37d647b5f5d8dbf68dfccde0 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 28 Aug 2026 20:32:01 -0400 Subject: [PATCH 55/78] DAOS-17519 test: Fixed the get_pool call. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 99164d32eae..7daa6165e25 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -38,11 +38,11 @@ def test_dlck_basic(self): """ dmg = self.get_dmg_command() self.log_step("Create a pool to run dlck") - pool = self.get_pool(self) + pool = self.get_pool() dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid dlck.log_dir = self.test_env.log_dir - dlck.storage = self.server_managers[0].get_config_value("scm_mount") + dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: dlck.nvme.value = os.path.join( self.server_managers[0].get_config_value("path"), "daos_control", "engine0", @@ -74,13 +74,13 @@ def test_dlck_basic_fault(self): self.log.info("Faults: %s", fault_list) self.log.info("Faults dict: %s", faults_dict) dmg = self.get_dmg_command() - pool = self.get_pool(self) + pool = self.get_pool() dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid dlck.log_dir = self.test_env.log_dir dlck.exit_status_exception = False dlck.env["D_FI_CONFIG"] = fault_inject_file - dlck.storage_mount.value = self.server_managers[0].get_config_value("scm_mount") + dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: dlck.nvme.value = os.path.join( self.server_managers[0].get_config_value("path"), "daos_control", "engine0", From e48754d68b677f6ab114728a586b7791766abc5b Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Sun, 30 Aug 2026 14:20:45 -0400 Subject: [PATCH 56/78] DAOS-17519 test: Renamed dlck_utils.py to dlck_test_base.py. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 22 +++++++++++++++---- .../util/{dlck_utils.py => dlck_test_base.py} | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) rename src/tests/ftest/util/{dlck_utils.py => dlck_test_base.py} (97%) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 7daa6165e25..4c2895d9a2c 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -6,18 +6,23 @@ import os import re -from dlck_utils import TestDlck +from dlck_test_base import DlckTestBase from fault_config_utils import FaultInjection from file_utils import distribute_files -class DlckBasicTest(TestDlck): +class DlckBasicTest(DlckTestBase): """Test class for dlck command line utility. :avocado: recursive """ def check_dlck_result(self, result, fault_name): - """Check that dlck reports the expected error for an injected fault.""" + """Check that dlck reported the expected error for an injected fault.""" + """ + Args: + result (dict): The result object returned by the dlck command. + fault_name (str): The name of the injected fault. + """ expected_errors = ("DER_INVAL", "DER_NONEXIST", "DER_ID_MISMATCH", "DER_DF_INCOMPT", "DER_DF_INVAL", "WARNING:") warning_pattern = r"\b\d+\s+warning(?:\(s\)|s?)\b" @@ -42,15 +47,20 @@ def test_dlck_basic(self): dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid dlck.log_dir = self.test_env.log_dir + dlck.run_user = 'daos_server' dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: dlck.nvme.value = os.path.join( self.server_managers[0].get_config_value("path"), "daos_control", "engine0", "daos_nvme.conf") + self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() + self.log_step("Run dlck command to check the health of the pool and storage") result = dlck.run() + + self.log_step("Perform dmg system start after running dlck command") dmg.system_start() if not result.passed: self.fail(f"dlck failed on {result.failed_hosts}") @@ -78,6 +88,7 @@ def test_dlck_basic_fault(self): dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid dlck.log_dir = self.test_env.log_dir + dlck.run_user = 'daos_server' dlck.exit_status_exception = False dlck.env["D_FI_CONFIG"] = fault_inject_file dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") @@ -85,8 +96,10 @@ def test_dlck_basic_fault(self): dlck.nvme.value = os.path.join( self.server_managers[0].get_config_value("path"), "daos_control", "engine0", "daos_nvme.conf") + self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() + for test_fault in fault_list: with open(fault_inject_file, 'w') as f: f.write("fault_config:\n") @@ -106,7 +119,8 @@ def test_dlck_basic_fault(self): error = self.check_dlck_result(result, test_fault) if error: errors.append(error) - self.log_step("Run the dmg start command") + + self.log_step("Perform dmg system start after running dlck command") dmg.system_start() if errors: diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_test_base.py similarity index 97% rename from src/tests/ftest/util/dlck_utils.py rename to src/tests/ftest/util/dlck_test_base.py index 00f32508b46..06ef1c6413c 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_test_base.py @@ -8,7 +8,7 @@ from command_utils_base import FormattedParameter -class TestDlck(TestWithServers): +class DlckTestBase(TestWithServers): # pylint: disable=too-few-public-methods """Base class for Dlck tests. From 0e72f41daa41a12e46d3475526ec0b4d6bcfbd0e Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Sun, 30 Aug 2026 14:30:19 -0400 Subject: [PATCH 57/78] DAOS-17519 test: Minor lint issue. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 4c2895d9a2c..1d7190d1d05 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -17,8 +17,7 @@ class DlckBasicTest(DlckTestBase): :avocado: recursive """ def check_dlck_result(self, result, fault_name): - """Check that dlck reported the expected error for an injected fault.""" - """ + """Check that dlck reported the expected error for an injected fault. Args: result (dict): The result object returned by the dlck command. fault_name (str): The name of the injected fault. From 3e608415cd39e2a251cfb202dbce1a5010192137 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Sun, 30 Aug 2026 21:32:28 -0400 Subject: [PATCH 58/78] DAOS-17519 test: Remove some of the fault injection flags. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 2 +- src/tests/ftest/recovery/dlck_basic.yaml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 1d7190d1d05..b648daa2272 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -24,7 +24,7 @@ def check_dlck_result(self, result, fault_name): """ expected_errors = ("DER_INVAL", "DER_NONEXIST", "DER_ID_MISMATCH", "DER_DF_INCOMPT", "DER_DF_INVAL", "WARNING:") - warning_pattern = r"\b\d+\s+warning(?:\(s\)|s?)\b" + warning_pattern = r"\b[1-9]\d*\s+warning(?:\(s\)|s?)\b" output = f"{result.joined_stdout}\n{result.joined_stderr}" expected_error_found = any(error in output for error in expected_errors) warning_found = re.search(warning_pattern, output) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index ab4b5b25471..fa9d806585c 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -16,10 +16,8 @@ pool: dlck_test_faults: fault_list: - - DLCK_MOCK_ROOT - DLCK_FAULT_GETGRNAM - DLCK_MOCK_NO_DAOS_SERVER_GROUP - - DLCK_MOCK_NOT_IN_DAOS_SERVER_GROUP - DLCK_FAULT_CREATE_POOL_DIR - DLCK_FAULT_CREATE_LOG_DIR - DLCK_FAULT_ENGINE_START From 36275190361edb0ba892c66563acbf500c56c292 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 31 Aug 2026 11:22:20 -0400 Subject: [PATCH 59/78] DAOS-17591 test: Add the log_dir and verbose parameter. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 4 ++-- src/tests/ftest/util/dlck_test_base.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index b648daa2272..416e35cb16d 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -45,7 +45,7 @@ def test_dlck_basic(self): pool = self.get_pool() dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid - dlck.log_dir = self.test_env.log_dir + dlck.log_dir.value = self.test_env.log_dir dlck.run_user = 'daos_server' dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: @@ -86,7 +86,7 @@ def test_dlck_basic_fault(self): pool = self.get_pool() dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid - dlck.log_dir = self.test_env.log_dir + dlck.log_dir.value = self.test_env.log_dir dlck.run_user = 'daos_server' dlck.exit_status_exception = False dlck.env["D_FI_CONFIG"] = fault_inject_file diff --git a/src/tests/ftest/util/dlck_test_base.py b/src/tests/ftest/util/dlck_test_base.py index 06ef1c6413c..b9dfbe05a6c 100644 --- a/src/tests/ftest/util/dlck_test_base.py +++ b/src/tests/ftest/util/dlck_test_base.py @@ -48,3 +48,4 @@ def __init__(self, path="", namespace="/run/dlck/*"): self.nvme = FormattedParameter("--nvme={}", None) self.storage = FormattedParameter("--storage={}", None) self.log_dir = FormattedParameter("--log_dir={}", None) + self.verbose = FormattedParameter("--verbose={}", True) From 4b8578d9a7aa93e3dc4e40356807e7cc6f928f9a Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 2 Sep 2026 18:54:08 -0400 Subject: [PATCH 60/78] DAOS-17519 test: Revert dlck_test_base to dlck_utils. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 18 ++--- src/tests/ftest/util/collection_utils.py | 13 +++- src/tests/ftest/util/command_utils.py | 68 ++++--------------- .../util/{dlck_test_base.py => dlck_utils.py} | 6 +- 4 files changed, 35 insertions(+), 70 deletions(-) rename src/tests/ftest/util/{dlck_test_base.py => dlck_utils.py} (92%) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 416e35cb16d..2d714a87556 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -6,12 +6,13 @@ import os import re -from dlck_test_base import DlckTestBase +import yaml +from dlck_utils import TestDlck from fault_config_utils import FaultInjection from file_utils import distribute_files -class DlckBasicTest(DlckTestBase): +class DlckBasicTest(TestDlck): """Test class for dlck command line utility. :avocado: recursive @@ -45,7 +46,7 @@ def test_dlck_basic(self): pool = self.get_pool() dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid - dlck.log_dir.value = self.test_env.log_dir + dlck.log_dir = self.test_env.log_dir dlck.run_user = 'daos_server' dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: @@ -100,14 +101,13 @@ def test_dlck_basic_fault(self): dmg.system_stop() for test_fault in fault_list: + fault_config = { + 'fault_config': [ + {key: str(value) for key, value in faults_dict[test_fault].items()}]} with open(fault_inject_file, 'w') as f: - f.write("fault_config:\n") - for count, (key, value) in enumerate(faults_dict[test_fault].items()): - if count == 0: - f.write(f"- {key}: \'{value}\'\n") - else: - f.write(f" {key}: \'{value}\'\n") + yaml.dump(fault_config, f) self.log.info("Reading the updated fault injection file contents") + with open(fault_inject_file, 'r') as f: file_data = f.read() self.log.info("\n %s", file_data) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 7c0d3ccf08d..7b9e0d3508b 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -30,7 +30,8 @@ FAILURE_TRIGGER = "00_trigger-launch-failure_00" TEST_EXPECT_CORE_FILES = ["./harness/core_files.py"] TEST_RESULTS_DIRS = ( - "daos_configs", "daos_logs", "cart_logs", "daos_dumps", "valgrind_logs", "stacktraces") + "daos_configs", "daos_logs", "cart_logs", "daos_dumps", "valgrind_logs", "stacktraces", + "dlck_check") def stop_daos_agent_services(logger, test): @@ -1047,6 +1048,14 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "depth": 1, "timeout": 900, } + remote_files["dlck files"] = { + "source": test_env.log_dir, + "destination": os.path.join(job_results_dir, "latest", TEST_RESULTS_DIRS[6]), + "pattern": "dlck_check*", + "hosts": test.host_info.all_hosts, + "depth": 1, + "timeout": 900, + } for index, hosts in enumerate(core_files): remote_files[f"core files {index + 1}/{len(core_files)}"] = { "source": core_files[hosts]["path"], diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 2b0ed39bc5e..6a33ba0d8d9 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -267,7 +267,7 @@ def check_results(self): if self.result and self.check_results_list: regex = r"({})".format("|".join(self.check_results_list)) self.log.debug("Checking the command output for any bad keywords: %s", regex) - for output in (self._result_stdout(), self._result_stderr()): + for output in (self.__result_stdout(), self.__result_stderr()): match = re.findall(regex, output) if match: self.log.info( @@ -279,24 +279,20 @@ def check_results(self): break return status - def _result_stdout(self): + def __result_stdout(self): """Get all the stdout from the command result. Returns: str: the command result's stdout as a string """ - if not self.result: - raise CommandFailure("No command result available to return stdout") return self.result.stdout_text - def _result_stderr(self): + def __result_stderr(self): """Get all the stderr from the command result. Returns: str: the command result's stderr as a string """ - if not self.result: - raise CommandFailure("No command result available to return stderr") return self.result.stderr_text def _run_subprocess(self): @@ -410,9 +406,7 @@ def cleanup_command(self): regex = self.command_regex if self.full_command_regex: regex = f"'{str(self)}'" - hosts = None - if hasattr(self, "hosts"): - hosts = self.hosts + hosts = getattr(self, "hosts", None) detected, running = stop_processes( self.log, hosts, regex, full_command=self.full_command_regex) if not detected: @@ -473,40 +467,6 @@ def get_subprocess_state(self, message=None): r"\d+\s+([DRSTtWXZ Date: Tue, 8 Sep 2026 11:37:02 -0400 Subject: [PATCH 61/78] DAOS-17519 test: Make daos_server as default run_user. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 2 -- src/tests/ftest/util/dlck_utils.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 2d714a87556..c43546bf4dc 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -47,7 +47,6 @@ def test_dlck_basic(self): dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid dlck.log_dir = self.test_env.log_dir - dlck.run_user = 'daos_server' dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") if self.server_managers[0].manager.job.using_control_metadata: dlck.nvme.value = os.path.join( @@ -88,7 +87,6 @@ def test_dlck_basic_fault(self): dlck = self.get_dlck_command() dlck.pool_uuid.value = pool.uuid dlck.log_dir.value = self.test_env.log_dir - dlck.run_user = 'daos_server' dlck.exit_status_exception = False dlck.env["D_FI_CONFIG"] = fault_inject_file dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 7a3c174b42a..0607be057d5 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -29,6 +29,7 @@ def get_dlck_command(self, path="", namespace="/run/dlck/*"): dlck.register_cleanup_method = self.register_cleanup dlck.hosts = self.server_managers[0].hosts[0:1] dlck.log_dir.value = self.log_dir + dlck.run_user = 'daos_server' dlck.get_params(self) return dlck From e7f30a4619b209fb058c8f89e8ec703852b0fbd1 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 9 Sep 2026 00:15:25 -0400 Subject: [PATCH 62/78] DAOS-17519 test: Updated archive scripts for dlck. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/collection_utils.py | 120 ++++++++++++++++++++--- src/tests/ftest/util/run_utils.py | 5 +- 2 files changed, 110 insertions(+), 15 deletions(-) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 7b9e0d3508b..b673ef05ae4 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -225,8 +225,70 @@ def check_server_storage(logger, test, test_result, stage): return status +def archive_files_in_dir(logger, summary, hosts, source, dir_pattern, file_pattern, + destination, depth, threshold, timeout, test_result, + test=None, dir_chown=False): + """ + Archive files in directories matching the specified pattern on remote hosts. + + Args: + logger (logging.Logger): Logger for output messages. + summary (str): Summary of the operation. + hosts (set): Set of remote hosts. + source (str): Source directory on remote hosts. + dir_pattern (str): Pattern to match directories. + file_pattern (str): Pattern to match files within the directories. + destination (str): Destination directory for archived files. + depth (int): Depth for archiving files. + threshold (int): Threshold for archiving. + timeout (int): Timeout for archiving operation. + test_result (TestResult): Test result object for reporting failures. + test (optional): Test object, if applicable. + dir_chown (bool): Whether to change ownership of directories before archiving. + + Returns: + int: Return code indicating success or failure of the operation. + """ + if depth is None: + depth = 1 + + logger.debug("=" * 80) + logger.info( + "Archiving %s from %s:%s to %s%s,", + summary, hosts, os.path.join(source, dir_pattern, file_pattern), destination, + f" after running '{str(test)}'" if test else "") + logger.debug(" Remote hosts: %s", hosts.difference(get_local_host())) + logger.debug(" Local host: %s", hosts.intersection(get_local_host())) + + # List any remote directories and determine which hosts contain these directories + return_code, dir_hosts = list_files(logger, hosts, source, dir_pattern, depth, + test_result, file_type="d") + if not dir_hosts: + # If no directories found then there is nothing else to do + logger.debug("No %s directories found on %s", os.path.join(source, dir_pattern), hosts) + return return_code + + # Change the ownership of matching directories if requested + if dir_chown: + other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()] + result = run_remote(logger, hosts, find_command(source, dir_pattern, depth, other, + file_type="d")) + if not result.passed: + message = (f"Error changing {os.path.join(source, dir_pattern)} file permissions on " + f"{result.failed_hosts}") + test_result.fail_test(logger, "Process", message) + return_code = 16 + hosts = result.passed_hosts.copy() + + # Now archive the files in each directory + return_code |= archive_files(logger, summary, hosts, os.path.join(source, dir_pattern), + file_pattern, destination, depth, threshold, timeout, + test_result, test=None, chown=dir_chown) + return return_code + + def archive_files(logger, summary, hosts, source, pattern, destination, depth, threshold, timeout, - test_result, test=None): + test_result, test=None, chown=False): # pylint: disable=too-many-arguments """Archive the files from the source to the destination. @@ -242,6 +304,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t timeout (int): number of seconds to wait for the command to complete. test_result (TestResult): the test result used to update the status of the test test (TestInfo, optional): the test information. Defaults to None. + chown (bool, optional): whether to change file ownership before moving. Defaults to False. Returns: int: status code: 0 = success, 16 = failure @@ -255,6 +318,17 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t logger.debug(" Remote hosts: %s", hosts.difference(get_local_host())) logger.debug(" Local host: %s", hosts.intersection(get_local_host())) + if chown: + other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()] + result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) + if not result.passed: + message = (f"Error changing {os.path.join(source, pattern)} file permissions on " + f"{result.failed_hosts}") + test_result.fail_test(logger, "Process", message) + return_code = 16 + hosts = result.passed_hosts.copy() + return return_code + # List any remote files and their sizes and determine which hosts contain these files return_code, file_hosts = list_files(logger, hosts, source, pattern, depth, test_result) if not file_hosts: @@ -282,7 +356,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t # Move the test files to the test-results directory on this host return_code |= move_files( - logger, file_hosts, source, pattern, destination, depth, timeout, test_result) + logger, file_hosts, source, pattern, destination, depth, timeout, test_result, chown=chown) if test and "core files" in summary: # Process the core files @@ -291,7 +365,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t return return_code -def list_files(logger, hosts, source, pattern, depth, test_result): +def list_files(logger, hosts, source, pattern, depth, test_result, file_type='f'): """List the files in source with that match the pattern. Args: @@ -301,6 +375,7 @@ def list_files(logger, hosts, source, pattern, depth, test_result): pattern (str): pattern used to limit which files are processed depth (int): max depth for find command test_result (TestResult): the test result used to update the status of the test + file_type (str): type of files to list, default is 'f' for regular files Returns: tuple: a tuple containing: @@ -314,7 +389,7 @@ def list_files(logger, hosts, source, pattern, depth, test_result): logger.debug("-" * 80) logger.debug("Listing any %s files on %s", source_files, hosts) other = ["-printf", "'%M %n %-12u %-12g %12k %t %p\n'"] - result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) + result = run_remote(logger, hosts, find_command(source, pattern, depth, other, file_type)) if not result.passed: message = f"Error determining if {source_files} files exist on {result.failed_hosts}" test_result.fail_test(logger, "Process", message) @@ -322,7 +397,7 @@ def list_files(logger, hosts, source, pattern, depth, test_result): else: for data in result.output: for line in data.stdout: - if source in line: + if (source in line) or re.search(source, line): logger.debug("Found at least one file match on %s: %s", data.hosts, line) hosts_with_files.add(data.hosts) break @@ -467,7 +542,8 @@ def compress_files(logger, hosts, source, pattern, depth, test_result): return 0 -def move_files(logger, hosts, source, pattern, destination, depth, timeout, test_result): +def move_files(logger, hosts, source, pattern, destination, depth, + timeout, test_result, chown=False): """Move files from the source to the destination. Args: @@ -479,6 +555,7 @@ def move_files(logger, hosts, source, pattern, destination, depth, timeout, test depth (int): max depth for find command timeout (int): number of seconds to wait for the command to complete. test_result (TestResult): the test result used to update the status of the test + chown (bool, optional): whether to change file ownership before moving. Defaults to False. Returns: int: status code: 0 = success, 16 = failure @@ -488,8 +565,9 @@ def move_files(logger, hosts, source, pattern, destination, depth, timeout, test logger.debug("Moving files from %s to %s on %s", source, destination, hosts) return_code = 0 - # Core and dump files require a file ownership change before they can be copied - if "stacktrace" in destination or "daos_dumps" in destination: + # Core, dump and dlck files require a file ownership change before they can be copied + if chown is True and ("stacktrace" in destination + or "daos_dumps" in destination or re.search("dlck_check*", destination)): # pylint: disable=import-outside-toplevel other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()] result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) @@ -1006,6 +1084,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": get_local_host(), "depth": 1, "timeout": 300, + "chown": False, } for index, source in enumerate(test_env.config_file_directories()): remote_files[f"remote configuration files ({index})"] = { @@ -1015,6 +1094,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.all_hosts, "depth": 1, "timeout": 300, + "chown": False, } remote_files["daos log files"] = { "source": test_env.log_dir, @@ -1023,6 +1103,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.all_hosts, "depth": 1, "timeout": 900, + "chown": False, } remote_files["cart log files"] = { "source": test_env.log_dir, @@ -1031,6 +1112,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.all_hosts, "depth": 2, "timeout": 900, + "chown": False, } remote_files["ULTs stacks dump files"] = { "source": os.path.join(os.sep, "tmp"), @@ -1039,6 +1121,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.servers.hosts, "depth": 1, "timeout": 900, + "chown": True, } remote_files["valgrind log files"] = { "source": test_env.shared_dir, @@ -1047,14 +1130,16 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.servers.hosts, "depth": 1, "timeout": 900, + "chown": False, } remote_files["dlck files"] = { "source": test_env.log_dir, "destination": os.path.join(job_results_dir, "latest", TEST_RESULTS_DIRS[6]), - "pattern": "dlck_check*", + "pattern": "dlck_check*/*_vos-*", "hosts": test.host_info.all_hosts, "depth": 1, "timeout": 900, + "chown": True, } for index, hosts in enumerate(core_files): remote_files[f"core files {index + 1}/{len(core_files)}"] = { @@ -1064,14 +1149,23 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": NodeSet(hosts), "depth": 1, "timeout": 1800, + "chown": True, } for summary, data in remote_files.items(): if not data["hosts"]: continue - return_code |= archive_files( - logger, summary, data["hosts"].copy(), data["source"], data["pattern"], - data["destination"], data["depth"], threshold, data["timeout"], - test_result, test) + elif "dlck_check*/*_vos-*" in data["pattern"]: + split_data = data["pattern"].split("/") + return_code |= archive_files_in_dir( + logger, summary, data["hosts"].copy(), data["source"], + split_data[0], split_data[1], data["destination"], + data["depth"], threshold, data["timeout"], + test_result, test, data["chown"]) + else: + return_code |= archive_files( + logger, summary, data["hosts"].copy(), data["source"], data["pattern"], + data["destination"], data["depth"], threshold, data["timeout"], + test_result, test, data["chown"]) # Generate a steps.log file return_code |= create_steps_log(logger, job_results_dir, test_result) diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index e4785743912..6a48ecce9f0 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -519,7 +519,7 @@ def command_as_user(command, user, env=None): return " ".join(cmd_list) -def find_command(source, pattern, depth, other=None): +def find_command(source, pattern, depth, other=None, file_type="f"): """Get the find command. Args: @@ -528,11 +528,12 @@ def find_command(source, pattern, depth, other=None): depth (int): max depth for find command other (object, optional): other commands, as a list or str, to include at the end of the base find command. Defaults to None. + file_type (str, optional): type of files to find, default is 'f' for regular files. Returns: str: the find command """ - command = ["find", source, "-maxdepth", str(depth), "-type", "f", "-name", f"'{pattern}'"] + command = ["find", source, "-maxdepth", str(depth), "-type", file_type, "-name", f"'{pattern}'"] if isinstance(other, list): command.extend(other) elif isinstance(other, str): From 5451f8f471339abee0c053744e6ac52c1b262ad7 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 9 Sep 2026 00:30:01 -0400 Subject: [PATCH 63/78] DAOS-17519 test: Fix some pylint issues. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/collection_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index b673ef05ae4..1c3113f87d8 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -228,6 +228,7 @@ def check_server_storage(logger, test, test_result, stage): def archive_files_in_dir(logger, summary, hosts, source, dir_pattern, file_pattern, destination, depth, threshold, timeout, test_result, test=None, dir_chown=False): + # pylint: disable=too-many-arguments """ Archive files in directories matching the specified pattern on remote hosts. @@ -1154,11 +1155,11 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a for summary, data in remote_files.items(): if not data["hosts"]: continue - elif "dlck_check*/*_vos-*" in data["pattern"]: - split_data = data["pattern"].split("/") + if "dlck_check*/*_vos-*" in data["pattern"]: + dir_filename = data["pattern"].split("/") return_code |= archive_files_in_dir( logger, summary, data["hosts"].copy(), data["source"], - split_data[0], split_data[1], data["destination"], + dir_filename[0], dir_filename[1], data["destination"], data["depth"], threshold, data["timeout"], test_result, test, data["chown"]) else: From ad945283752195e3c738388b83c8ae5941ab3dfb Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 9 Sep 2026 10:55:37 -0400 Subject: [PATCH 64/78] DAOS-17519 test: Fix the verbose formatting. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/dlck_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py index 0607be057d5..02e569947bf 100644 --- a/src/tests/ftest/util/dlck_utils.py +++ b/src/tests/ftest/util/dlck_utils.py @@ -49,4 +49,4 @@ def __init__(self, path="", namespace="/run/dlck/*"): self.nvme = FormattedParameter("--nvme={}", None) self.storage = FormattedParameter("--storage={}", None) self.log_dir = FormattedParameter("--log_dir={}", None) - self.verbose = FormattedParameter("--verbose={}", True) + self.verbose = FormattedParameter("--verbose", True) From 2198f7c878bbf7f08ff8872c820c4dc3fce86a5f Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 9 Sep 2026 19:59:39 -0400 Subject: [PATCH 65/78] DAOS-17519 test: Reduce the complexity in dlck file archiving process. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/collection_utils.py | 129 ++++------------------- src/tests/ftest/util/run_utils.py | 5 +- 2 files changed, 24 insertions(+), 110 deletions(-) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 1c3113f87d8..d6744086f29 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -225,71 +225,8 @@ def check_server_storage(logger, test, test_result, stage): return status -def archive_files_in_dir(logger, summary, hosts, source, dir_pattern, file_pattern, - destination, depth, threshold, timeout, test_result, - test=None, dir_chown=False): - # pylint: disable=too-many-arguments - """ - Archive files in directories matching the specified pattern on remote hosts. - - Args: - logger (logging.Logger): Logger for output messages. - summary (str): Summary of the operation. - hosts (set): Set of remote hosts. - source (str): Source directory on remote hosts. - dir_pattern (str): Pattern to match directories. - file_pattern (str): Pattern to match files within the directories. - destination (str): Destination directory for archived files. - depth (int): Depth for archiving files. - threshold (int): Threshold for archiving. - timeout (int): Timeout for archiving operation. - test_result (TestResult): Test result object for reporting failures. - test (optional): Test object, if applicable. - dir_chown (bool): Whether to change ownership of directories before archiving. - - Returns: - int: Return code indicating success or failure of the operation. - """ - if depth is None: - depth = 1 - - logger.debug("=" * 80) - logger.info( - "Archiving %s from %s:%s to %s%s,", - summary, hosts, os.path.join(source, dir_pattern, file_pattern), destination, - f" after running '{str(test)}'" if test else "") - logger.debug(" Remote hosts: %s", hosts.difference(get_local_host())) - logger.debug(" Local host: %s", hosts.intersection(get_local_host())) - - # List any remote directories and determine which hosts contain these directories - return_code, dir_hosts = list_files(logger, hosts, source, dir_pattern, depth, - test_result, file_type="d") - if not dir_hosts: - # If no directories found then there is nothing else to do - logger.debug("No %s directories found on %s", os.path.join(source, dir_pattern), hosts) - return return_code - - # Change the ownership of matching directories if requested - if dir_chown: - other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()] - result = run_remote(logger, hosts, find_command(source, dir_pattern, depth, other, - file_type="d")) - if not result.passed: - message = (f"Error changing {os.path.join(source, dir_pattern)} file permissions on " - f"{result.failed_hosts}") - test_result.fail_test(logger, "Process", message) - return_code = 16 - hosts = result.passed_hosts.copy() - - # Now archive the files in each directory - return_code |= archive_files(logger, summary, hosts, os.path.join(source, dir_pattern), - file_pattern, destination, depth, threshold, timeout, - test_result, test=None, chown=dir_chown) - return return_code - - def archive_files(logger, summary, hosts, source, pattern, destination, depth, threshold, timeout, - test_result, test=None, chown=False): + test_result, test=None): # pylint: disable=too-many-arguments """Archive the files from the source to the destination. @@ -305,7 +242,6 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t timeout (int): number of seconds to wait for the command to complete. test_result (TestResult): the test result used to update the status of the test test (TestInfo, optional): the test information. Defaults to None. - chown (bool, optional): whether to change file ownership before moving. Defaults to False. Returns: int: status code: 0 = success, 16 = failure @@ -319,17 +255,6 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t logger.debug(" Remote hosts: %s", hosts.difference(get_local_host())) logger.debug(" Local host: %s", hosts.intersection(get_local_host())) - if chown: - other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()] - result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) - if not result.passed: - message = (f"Error changing {os.path.join(source, pattern)} file permissions on " - f"{result.failed_hosts}") - test_result.fail_test(logger, "Process", message) - return_code = 16 - hosts = result.passed_hosts.copy() - return return_code - # List any remote files and their sizes and determine which hosts contain these files return_code, file_hosts = list_files(logger, hosts, source, pattern, depth, test_result) if not file_hosts: @@ -357,7 +282,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t # Move the test files to the test-results directory on this host return_code |= move_files( - logger, file_hosts, source, pattern, destination, depth, timeout, test_result, chown=chown) + logger, file_hosts, source, pattern, destination, depth, timeout, test_result) if test and "core files" in summary: # Process the core files @@ -366,7 +291,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t return return_code -def list_files(logger, hosts, source, pattern, depth, test_result, file_type='f'): +def list_files(logger, hosts, source, pattern, depth, test_result): """List the files in source with that match the pattern. Args: @@ -376,7 +301,6 @@ def list_files(logger, hosts, source, pattern, depth, test_result, file_type='f' pattern (str): pattern used to limit which files are processed depth (int): max depth for find command test_result (TestResult): the test result used to update the status of the test - file_type (str): type of files to list, default is 'f' for regular files Returns: tuple: a tuple containing: @@ -390,7 +314,7 @@ def list_files(logger, hosts, source, pattern, depth, test_result, file_type='f' logger.debug("-" * 80) logger.debug("Listing any %s files on %s", source_files, hosts) other = ["-printf", "'%M %n %-12u %-12g %12k %t %p\n'"] - result = run_remote(logger, hosts, find_command(source, pattern, depth, other, file_type)) + result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) if not result.passed: message = f"Error determining if {source_files} files exist on {result.failed_hosts}" test_result.fail_test(logger, "Process", message) @@ -544,7 +468,7 @@ def compress_files(logger, hosts, source, pattern, depth, test_result): def move_files(logger, hosts, source, pattern, destination, depth, - timeout, test_result, chown=False): + timeout, test_result): """Move files from the source to the destination. Args: @@ -556,7 +480,6 @@ def move_files(logger, hosts, source, pattern, destination, depth, depth (int): max depth for find command timeout (int): number of seconds to wait for the command to complete. test_result (TestResult): the test result used to update the status of the test - chown (bool, optional): whether to change file ownership before moving. Defaults to False. Returns: int: status code: 0 = success, 16 = failure @@ -566,9 +489,8 @@ def move_files(logger, hosts, source, pattern, destination, depth, logger.debug("Moving files from %s to %s on %s", source, destination, hosts) return_code = 0 - # Core, dump and dlck files require a file ownership change before they can be copied - if chown is True and ("stacktrace" in destination - or "daos_dumps" in destination or re.search("dlck_check*", destination)): + # Core and dump files require a file ownership change before they can be copied + if "stacktrace" in destination or "daos_dumps" in destination: # pylint: disable=import-outside-toplevel other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()] result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) @@ -1077,6 +999,15 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a # size exceeding the threshold. test_env = TestEnvironment() if archive: + logger.debug("Making all '%s' files accessible for archiving", test_env.log_dir) + result = run_remote(logger, test.host_info.all_hosts, + "sudo -n {}".format(get_chown_command(options="-R", + file=test_env.log_dir))) + if not result.passed: + message = f"Error making {test_env.log_dir} files accessible for archiving" + test_result.fail_test(logger, "Process", message) + return_code = 16 + remote_files = OrderedDict() remote_files["local configuration files"] = { "source": test_env.log_dir, @@ -1085,7 +1016,6 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": get_local_host(), "depth": 1, "timeout": 300, - "chown": False, } for index, source in enumerate(test_env.config_file_directories()): remote_files[f"remote configuration files ({index})"] = { @@ -1095,7 +1025,6 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.all_hosts, "depth": 1, "timeout": 300, - "chown": False, } remote_files["daos log files"] = { "source": test_env.log_dir, @@ -1104,7 +1033,6 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.all_hosts, "depth": 1, "timeout": 900, - "chown": False, } remote_files["cart log files"] = { "source": test_env.log_dir, @@ -1113,7 +1041,6 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.all_hosts, "depth": 2, "timeout": 900, - "chown": False, } remote_files["ULTs stacks dump files"] = { "source": os.path.join(os.sep, "tmp"), @@ -1122,7 +1049,6 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.servers.hosts, "depth": 1, "timeout": 900, - "chown": True, } remote_files["valgrind log files"] = { "source": test_env.shared_dir, @@ -1131,16 +1057,14 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": test.host_info.servers.hosts, "depth": 1, "timeout": 900, - "chown": False, } remote_files["dlck files"] = { "source": test_env.log_dir, "destination": os.path.join(job_results_dir, "latest", TEST_RESULTS_DIRS[6]), - "pattern": "dlck_check*/*_vos-*", + "pattern": "*_vos-*", "hosts": test.host_info.all_hosts, - "depth": 1, + "depth": 2, "timeout": 900, - "chown": True, } for index, hosts in enumerate(core_files): remote_files[f"core files {index + 1}/{len(core_files)}"] = { @@ -1150,23 +1074,14 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a "hosts": NodeSet(hosts), "depth": 1, "timeout": 1800, - "chown": True, } for summary, data in remote_files.items(): if not data["hosts"]: continue - if "dlck_check*/*_vos-*" in data["pattern"]: - dir_filename = data["pattern"].split("/") - return_code |= archive_files_in_dir( - logger, summary, data["hosts"].copy(), data["source"], - dir_filename[0], dir_filename[1], data["destination"], - data["depth"], threshold, data["timeout"], - test_result, test, data["chown"]) - else: - return_code |= archive_files( - logger, summary, data["hosts"].copy(), data["source"], data["pattern"], - data["destination"], data["depth"], threshold, data["timeout"], - test_result, test, data["chown"]) + return_code |= archive_files( + logger, summary, data["hosts"].copy(), data["source"], data["pattern"], + data["destination"], data["depth"], threshold, data["timeout"], + test_result, test) # Generate a steps.log file return_code |= create_steps_log(logger, job_results_dir, test_result) diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index 6a48ecce9f0..e4785743912 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -519,7 +519,7 @@ def command_as_user(command, user, env=None): return " ".join(cmd_list) -def find_command(source, pattern, depth, other=None, file_type="f"): +def find_command(source, pattern, depth, other=None): """Get the find command. Args: @@ -528,12 +528,11 @@ def find_command(source, pattern, depth, other=None, file_type="f"): depth (int): max depth for find command other (object, optional): other commands, as a list or str, to include at the end of the base find command. Defaults to None. - file_type (str, optional): type of files to find, default is 'f' for regular files. Returns: str: the find command """ - command = ["find", source, "-maxdepth", str(depth), "-type", file_type, "-name", f"'{pattern}'"] + command = ["find", source, "-maxdepth", str(depth), "-type", "f", "-name", f"'{pattern}'"] if isinstance(other, list): command.extend(other) elif isinstance(other, str): From 6c8b6a4f10611db397bfce0aa69e85c5e4ea5c73 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 10 Sep 2026 14:42:09 -0400 Subject: [PATCH 66/78] DAOS-17519 test: Archive the dlck log directories. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 2 +- src/tests/ftest/util/collection_utils.py | 35 ++++++++++++++---------- src/tests/ftest/util/run_utils.py | 5 ++-- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index c43546bf4dc..0ef4c9be06f 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -24,7 +24,7 @@ def check_dlck_result(self, result, fault_name): fault_name (str): The name of the injected fault. """ expected_errors = ("DER_INVAL", "DER_NONEXIST", "DER_ID_MISMATCH", - "DER_DF_INCOMPT", "DER_DF_INVAL", "WARNING:") + "DER_DF_INCOMPT", "DER_DF_INVAL", "WARNING:", "DER_PROTO") warning_pattern = r"\b[1-9]\d*\s+warning(?:\(s\)|s?)\b" output = f"{result.joined_stdout}\n{result.joined_stderr}" expected_error_found = any(error in output for error in expected_errors) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index d6744086f29..e680be03da9 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -226,7 +226,7 @@ def check_server_storage(logger, test, test_result, stage): def archive_files(logger, summary, hosts, source, pattern, destination, depth, threshold, timeout, - test_result, test=None): + test_result, test=None, file_type="f"): # pylint: disable=too-many-arguments """Archive the files from the source to the destination. @@ -242,6 +242,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t timeout (int): number of seconds to wait for the command to complete. test_result (TestResult): the test result used to update the status of the test test (TestInfo, optional): the test information. Defaults to None. + file_type (str, optional): type of file to search for. Defaults to "f". Returns: int: status code: 0 = success, 16 = failure @@ -256,7 +257,8 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t logger.debug(" Local host: %s", hosts.intersection(get_local_host())) # List any remote files and their sizes and determine which hosts contain these files - return_code, file_hosts = list_files(logger, hosts, source, pattern, depth, test_result) + return_code, file_hosts = list_files(logger, hosts, source, pattern, depth, + test_result, file_type) if not file_hosts: # If no files are found then there is nothing else to do logger.debug("No %s files found on %s", os.path.join(source, pattern), hosts) @@ -282,7 +284,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t # Move the test files to the test-results directory on this host return_code |= move_files( - logger, file_hosts, source, pattern, destination, depth, timeout, test_result) + logger, file_hosts, source, pattern, destination, depth, timeout, test_result, file_type) if test and "core files" in summary: # Process the core files @@ -291,16 +293,17 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t return return_code -def list_files(logger, hosts, source, pattern, depth, test_result): +def list_files(logger, hosts, source, pattern, depth, test_result, file_type="f"): """List the files in source with that match the pattern. Args: logger (Logger): logger for the messages produced by this method - hosts (NodSet): hosts on which the files are located + hosts (NodeSet): hosts on which the files are located source (str): where the files are currently located pattern (str): pattern used to limit which files are processed depth (int): max depth for find command test_result (TestResult): the test result used to update the status of the test + file_type (str, optional): type of file to search for. Defaults to "f". Returns: tuple: a tuple containing: @@ -314,7 +317,8 @@ def list_files(logger, hosts, source, pattern, depth, test_result): logger.debug("-" * 80) logger.debug("Listing any %s files on %s", source_files, hosts) other = ["-printf", "'%M %n %-12u %-12g %12k %t %p\n'"] - result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) + result = run_remote(logger, hosts, find_command(source, pattern, depth, + other, file_type=file_type)) if not result.passed: message = f"Error determining if {source_files} files exist on {result.failed_hosts}" test_result.fail_test(logger, "Process", message) @@ -468,18 +472,19 @@ def compress_files(logger, hosts, source, pattern, depth, test_result): def move_files(logger, hosts, source, pattern, destination, depth, - timeout, test_result): + timeout, test_result, file_type="f"): """Move files from the source to the destination. Args: logger (Logger): logger for the messages produced by this method - hosts (NodSet): hosts on which the files are located + hosts (NodeSet): hosts on which the files are located source (str): where the files are currently located pattern (str): pattern used to limit which files are processed destination (str): where the files should be moved to on this host depth (int): max depth for find command timeout (int): number of seconds to wait for the command to complete. test_result (TestResult): the test result used to update the status of the test + file_type (str, optional): type of file to search for. Defaults to "f". Returns: int: status code: 0 = success, 16 = failure @@ -493,7 +498,8 @@ def move_files(logger, hosts, source, pattern, destination, depth, if "stacktrace" in destination or "daos_dumps" in destination: # pylint: disable=import-outside-toplevel other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()] - result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) + result = run_remote(logger, hosts, find_command(source, pattern, depth, + other, file_type=file_type)) if not result.passed: message = (f"Error changing {os.path.join(source, pattern)} file permissions on " f"{result.failed_hosts}") @@ -530,8 +536,8 @@ def move_files(logger, hosts, source, pattern, destination, depth, return return_code # Move all the source files matching the pattern into the temporary remote directory - other = f"-print0 | xargs -0 -r0 -I '{{}}' {sudo_command}mv '{{}}' '{tmp_copy_dir}'/" - result = run_remote(logger, hosts, find_command(source, pattern, depth, other)) + other = (f"-print0 | xargs -0 -r0 -I '{{}}' {sudo_command} mv '{{}}' '{tmp_copy_dir}'/") + result = run_remote(logger, hosts, find_command(source, pattern, depth, other, file_type)) if not result.passed: message = (f"Error moving files to temporary remote copy directory '{tmp_copy_dir}' on " f"{result.failed_hosts}") @@ -1061,9 +1067,10 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a remote_files["dlck files"] = { "source": test_env.log_dir, "destination": os.path.join(job_results_dir, "latest", TEST_RESULTS_DIRS[6]), - "pattern": "*_vos-*", + "pattern": "dlck_check_*", "hosts": test.host_info.all_hosts, - "depth": 2, + "depth": 1, + "file_type": "d", "timeout": 900, } for index, hosts in enumerate(core_files): @@ -1081,7 +1088,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a return_code |= archive_files( logger, summary, data["hosts"].copy(), data["source"], data["pattern"], data["destination"], data["depth"], threshold, data["timeout"], - test_result, test) + test_result, test, file_type=data.get("file_type", "f")) # Generate a steps.log file return_code |= create_steps_log(logger, job_results_dir, test_result) diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index e4785743912..33bf1c1649d 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -519,7 +519,7 @@ def command_as_user(command, user, env=None): return " ".join(cmd_list) -def find_command(source, pattern, depth, other=None): +def find_command(source, pattern, depth, other=None, file_type="f"): """Get the find command. Args: @@ -528,11 +528,12 @@ def find_command(source, pattern, depth, other=None): depth (int): max depth for find command other (object, optional): other commands, as a list or str, to include at the end of the base find command. Defaults to None. + file_type (str, optional): type of file to search for. Defaults to "f". Returns: str: the find command """ - command = ["find", source, "-maxdepth", str(depth), "-type", "f", "-name", f"'{pattern}'"] + command = ["find", source, "-maxdepth", str(depth), "-type", file_type, "-name", f"'{pattern}'"] if isinstance(other, list): command.extend(other) elif isinstance(other, str): From 6892c0ab40950e332c59adb96d93acaf285b2d49 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 17 Sep 2026 12:06:36 -0400 Subject: [PATCH 67/78] DAOS-17519 test: Remove DLCK_FAULT_ENGINE_START/STOP fault flag. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index fa9d806585c..d8af4476b34 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -21,8 +21,6 @@ dlck_test_faults: - DLCK_FAULT_CREATE_POOL_DIR - DLCK_FAULT_CREATE_LOG_DIR - DLCK_FAULT_ENGINE_START - - DLCK_FAULT_ENGINE_JOIN - - DLCK_FAULT_ENGINE_STOP - DAOS_FAULT_POOL_NVME_HEALTH - DAOS_FAULT_POOL_OPEN_BIO - DAOS_FAULT_POOL_OPEN_UMEM From 003343d7dac1cffbac0282c96de0bf17b5434295 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 17 Sep 2026 12:17:53 -0400 Subject: [PATCH 68/78] DAOS-17519 test: Minor pylint issue. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/collection_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index e680be03da9..6e0fc7ea668 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -536,7 +536,7 @@ def move_files(logger, hosts, source, pattern, destination, depth, return return_code # Move all the source files matching the pattern into the temporary remote directory - other = (f"-print0 | xargs -0 -r0 -I '{{}}' {sudo_command} mv '{{}}' '{tmp_copy_dir}'/") + other = f"-print0 | xargs -0 -r0 -I '{{}}' {sudo_command} mv '{{}}' '{tmp_copy_dir}'/" result = run_remote(logger, hosts, find_command(source, pattern, depth, other, file_type)) if not result.passed: message = (f"Error moving files to temporary remote copy directory '{tmp_copy_dir}' on " From b9d170d1181f095be6ca2cb74dfec3cc709c1352 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 17 Sep 2026 12:20:46 -0400 Subject: [PATCH 69/78] DAOS-17519 test: Missed some review updates. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/util/collection_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 6e0fc7ea668..62a6397fa6b 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -318,7 +318,7 @@ def list_files(logger, hosts, source, pattern, depth, test_result, file_type="f" logger.debug("Listing any %s files on %s", source_files, hosts) other = ["-printf", "'%M %n %-12u %-12g %12k %t %p\n'"] result = run_remote(logger, hosts, find_command(source, pattern, depth, - other, file_type=file_type)) + other, file_type)) if not result.passed: message = f"Error determining if {source_files} files exist on {result.failed_hosts}" test_result.fail_test(logger, "Process", message) @@ -499,7 +499,7 @@ def move_files(logger, hosts, source, pattern, destination, depth, # pylint: disable=import-outside-toplevel other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()] result = run_remote(logger, hosts, find_command(source, pattern, depth, - other, file_type=file_type)) + other, file_type)) if not result.passed: message = (f"Error changing {os.path.join(source, pattern)} file permissions on " f"{result.failed_hosts}") From d80c44b35228bba85cf766bf133d75eb8fc96675 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 17 Sep 2026 18:56:15 -0400 Subject: [PATCH 70/78] DAOS-17519 test: Increasing the pool size to check whether MDonSSD works. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index d8af4476b34..c1b5356e821 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -12,7 +12,7 @@ server_config: storage: auto pool: - size: 10% + size: 60% dlck_test_faults: fault_list: @@ -22,7 +22,7 @@ dlck_test_faults: - DLCK_FAULT_CREATE_LOG_DIR - DLCK_FAULT_ENGINE_START - DAOS_FAULT_POOL_NVME_HEALTH - - DAOS_FAULT_POOL_OPEN_BIO + # - DAOS_FAULT_POOL_OPEN_BIO - DAOS_FAULT_POOL_OPEN_UMEM - DAOS_FAULT_POOL_OPEN_MAGIC - DAOS_FAULT_POOL_OPEN_VERSION From 3aa10ef07b3de94062f1682b2fad0a631910640c Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Fri, 18 Sep 2026 07:03:06 -0400 Subject: [PATCH 71/78] DAOS-17519 test: Add debug commands: mount, df Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 5 +++++ src/tests/ftest/recovery/dlck_basic.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 0ef4c9be06f..47d36ebc467 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -10,6 +10,7 @@ from dlck_utils import TestDlck from fault_config_utils import FaultInjection from file_utils import distribute_files +from run_utils import run_remote class DlckBasicTest(TestDlck): @@ -55,6 +56,8 @@ def test_dlck_basic(self): self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() + run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") + run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h") self.log_step("Run dlck command to check the health of the pool and storage") result = dlck.run() @@ -97,6 +100,8 @@ def test_dlck_basic_fault(self): self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() + run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") + run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h") for test_fault in fault_list: fault_config = { diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index c1b5356e821..d855a304ef8 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -12,7 +12,7 @@ server_config: storage: auto pool: - size: 60% + size: 30% dlck_test_faults: fault_list: From 0c49e51ee1d0bce367d73618306750fdca2805c4 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 1 Oct 2026 08:46:54 -0400 Subject: [PATCH 72/78] DAOS-17519 test: Update the dlck_basic.yaml. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index d855a304ef8..9b9f4a78416 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -22,7 +22,6 @@ dlck_test_faults: - DLCK_FAULT_CREATE_LOG_DIR - DLCK_FAULT_ENGINE_START - DAOS_FAULT_POOL_NVME_HEALTH - # - DAOS_FAULT_POOL_OPEN_BIO - DAOS_FAULT_POOL_OPEN_UMEM - DAOS_FAULT_POOL_OPEN_MAGIC - DAOS_FAULT_POOL_OPEN_VERSION From 8bcc6c2a47ab629a1b04ff9aac86d6202a937570 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Thu, 1 Oct 2026 14:24:52 -0400 Subject: [PATCH 73/78] DAOS-17519 test: Re-create tmpfs mount for MDonSSD environment. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Test-repeat: 3 Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 47d36ebc467..ef5da8d9f9d 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -100,8 +100,13 @@ def test_dlck_basic_fault(self): self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() - run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") - run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h") + if self.server_managers[0].manager.job.using_control_metadata: + # Mount 128G of memory-backed tmpfs for testing purposes + cmd = "mount -t tmpfs -o size=128G tmpfs /mnt/daos0" + run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd) + # Display the tmpfs mount + run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") + run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h") for test_fault in fault_list: fault_config = { From 3badfd2f8739cf69cf8eac4caa0c9734a259c5d9 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 7 Oct 2026 09:16:52 -0400 Subject: [PATCH 74/78] DAOS-17519 test: Update the storage mount to dlck_test. Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index ef5da8d9f9d..26d27e1866a 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -97,16 +97,16 @@ def test_dlck_basic_fault(self): dlck.nvme.value = os.path.join( self.server_managers[0].get_config_value("path"), "daos_control", "engine0", "daos_nvme.conf") - - self.log_step("Perform dmg system stop to run dlck command") - dmg.system_stop() - if self.server_managers[0].manager.job.using_control_metadata: # Mount 128G of memory-backed tmpfs for testing purposes - cmd = "mount -t tmpfs -o size=128G tmpfs /mnt/daos0" + cmd = "mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test" run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd) # Display the tmpfs mount run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h") + dlck.storage.value = "/mnt/dlck_test" + + self.log_step("Perform dmg system stop to run dlck command") + dmg.system_stop() for test_fault in fault_list: fault_config = { From 6534a44df8cb339af1d62242abf683493625dda8 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Mon, 19 Oct 2026 15:37:52 -0400 Subject: [PATCH 75/78] DAOS-17519 test: Enable DAOS_FAULT_POOL_OPEN_BIO for MDonSSD. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Test-repeat: 2 Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 11 ++++++++--- src/tests/ftest/recovery/dlck_basic.yaml | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 26d27e1866a..a56dbee8924 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -93,6 +93,10 @@ def test_dlck_basic_fault(self): dlck.exit_status_exception = False dlck.env["D_FI_CONFIG"] = fault_inject_file dlck.storage.value = self.server_managers[0].get_config_value("scm_mount") + + self.log_step("Perform dmg system stop to run dlck command") + dmg.system_stop() + if self.server_managers[0].manager.job.using_control_metadata: dlck.nvme.value = os.path.join( self.server_managers[0].get_config_value("path"), "daos_control", "engine0", @@ -105,10 +109,11 @@ def test_dlck_basic_fault(self): run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h") dlck.storage.value = "/mnt/dlck_test" - self.log_step("Perform dmg system stop to run dlck command") - dmg.system_stop() - for test_fault in fault_list: + if test_fault == "DAOS_FAULT_POOL_OPEN_BIO" and \ + not self.server_managers[0].manager.job.using_control_metadata: + self.log.info("Skipping test fault %s as control metadata is not used", test_fault) + continue fault_config = { 'fault_config': [ {key: str(value) for key, value in faults_dict[test_fault].items()}]} diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml index 9b9f4a78416..d2ed654876b 100644 --- a/src/tests/ftest/recovery/dlck_basic.yaml +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -34,3 +34,4 @@ dlck_test_faults: - DAOS_FAULT_BTREE_FEATURES_29 - DAOS_FAULT_POOL_EXT_PADDING - DAOS_FAULT_POOL_EXT_RESERVED + - DAOS_FAULT_POOL_OPEN_BIO From f8a72ea6489ce1bce24113632dbf15fb67f5e0d7 Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 20 Oct 2026 08:40:34 -0400 Subject: [PATCH 76/78] DAOS-17519 test: mkdir /mnt/dlck_test directory. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Test-repeat: 2 Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index a56dbee8924..727c4acaa87 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -56,8 +56,18 @@ def test_dlck_basic(self): self.log_step("Perform dmg system stop to run dlck command") dmg.system_stop() - run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") - run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h") + + if self.server_managers[0].manager.job.using_control_metadata: + dlck.nvme.value = os.path.join( + self.server_managers[0].get_config_value("path"), "daos_control", "engine0", + "daos_nvme.conf") + # Mount 128G of memory-backed tmpfs for testing purposes + cmd = "mkdir -p /mnt/dlck_test && mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test" + run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd) + # Display the tmpfs mount + run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") + run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h") + dlck.storage.value = "/mnt/dlck_test" self.log_step("Run dlck command to check the health of the pool and storage") result = dlck.run() @@ -102,7 +112,7 @@ def test_dlck_basic_fault(self): self.server_managers[0].get_config_value("path"), "daos_control", "engine0", "daos_nvme.conf") # Mount 128G of memory-backed tmpfs for testing purposes - cmd = "mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test" + cmd = "mkdir -p /mnt/dlck_test && mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test" run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd) # Display the tmpfs mount run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") From 1a3aaeb0883af5389e2cd305033748a0fab8669e Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Tue, 20 Oct 2026 21:45:59 -0400 Subject: [PATCH 77/78] DAOS-17519 test: Add sudo to mount command. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 727c4acaa87..18792c826a1 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -62,7 +62,8 @@ def test_dlck_basic(self): self.server_managers[0].get_config_value("path"), "daos_control", "engine0", "daos_nvme.conf") # Mount 128G of memory-backed tmpfs for testing purposes - cmd = "mkdir -p /mnt/dlck_test && mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test" + cmd = 'sudo -n "mkdir -p /mnt/dlck_test &&' \ + 'mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test"' run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd) # Display the tmpfs mount run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") @@ -112,7 +113,8 @@ def test_dlck_basic_fault(self): self.server_managers[0].get_config_value("path"), "daos_control", "engine0", "daos_nvme.conf") # Mount 128G of memory-backed tmpfs for testing purposes - cmd = "mkdir -p /mnt/dlck_test && mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test" + cmd = 'sudo -n "mkdir -p /mnt/dlck_test &&' \ + 'mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test"' run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd) # Display the tmpfs mount run_remote(log=self.log, hosts=self.hostlist_servers, command="mount") From 4d19ea97fe9fc19a3ff78cb80c1d5cf70564288e Mon Sep 17 00:00:00 2001 From: rpadma2 Date: Wed, 21 Oct 2026 11:56:10 -0400 Subject: [PATCH 78/78] DAOS-17519 test: Typo issue while executing the mount command. Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-fault-injection-test: true Skip-unit-test: true Skip-nlt: true Test-tag: DlckBasicFaultTest DlckBasicTest Signed-off-by: rpadma2 --- src/tests/ftest/recovery/dlck_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py index 18792c826a1..073ebdcc10d 100644 --- a/src/tests/ftest/recovery/dlck_basic.py +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -62,7 +62,7 @@ def test_dlck_basic(self): self.server_managers[0].get_config_value("path"), "daos_control", "engine0", "daos_nvme.conf") # Mount 128G of memory-backed tmpfs for testing purposes - cmd = 'sudo -n "mkdir -p /mnt/dlck_test &&' \ + cmd = 'sudo "mkdir -p /mnt/dlck_test && ' \ 'mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test"' run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd) # Display the tmpfs mount @@ -113,7 +113,7 @@ def test_dlck_basic_fault(self): self.server_managers[0].get_config_value("path"), "daos_control", "engine0", "daos_nvme.conf") # Mount 128G of memory-backed tmpfs for testing purposes - cmd = 'sudo -n "mkdir -p /mnt/dlck_test &&' \ + cmd = 'sudo "mkdir -p /mnt/dlck_test && ' \ 'mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test"' run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd) # Display the tmpfs mount