DAOS-17519 test: Automate dlck testing (basic/fault_injection)

src/tests/ftest/recovery/dlck_basic.py

@@ -0,0 +1,152 @@
+"""
+  Copyright 2026 Hewlett Packard Enterprise Development LP
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+"""
+import os
+import re
+
+import yaml
+from dlck_utils import TestDlck
+from fault_config_utils import FaultInjection
+from file_utils import distribute_files
+from run_utils import run_remote
+
+
+class DlckBasicTest(TestDlck):
+    """Test class for dlck command line utility.
+
+    :avocado: recursive
+    """
+    def check_dlck_result(self, result, fault_name):
+        """Check that dlck reported the expected error for an injected fault.
+        Args:
+            result (dict): The result object returned by the dlck command.
+            fault_name (str): The name of the injected fault.
+        """
+        expected_errors = ("DER_INVAL", "DER_NONEXIST", "DER_ID_MISMATCH",
+                           "DER_DF_INCOMPT", "DER_DF_INVAL", "WARNING:", "DER_PROTO")
+        warning_pattern = r"\b[1-9]\d*\s+warning(?:\(s\)|s?)\b"
+        output = f"{result.joined_stdout}\n{result.joined_stderr}"
+        expected_error_found = any(error in output for error in expected_errors)
+        warning_found = re.search(warning_pattern, output)
+        if not (expected_error_found or warning_found):
+            return f"{fault_name}: no expected error found in dlck output: {output}"
+        return None
+
+    def test_dlck_basic(self):
+        """Basic Test: Run 'dlck' command
+
+        :avocado: tags=all,daily_regression
+        :avocado: tags=hw,medium
+        :avocado: tags=recovery,dlck_cmd
+        :avocado: tags=DlckBasicTest,test_dlck_basic
+        """
+        dmg = self.get_dmg_command()
+        self.log_step("Create a pool to run dlck")
+        pool = self.get_pool()
+        dlck = self.get_dlck_command()
+        dlck.pool_uuid.value = pool.uuid
+        dlck.log_dir = self.test_env.log_dir
+        dlck.storage.value = self.server_managers[0].get_config_value("scm_mount")
+        if self.server_managers[0].manager.job.using_control_metadata:
+            dlck.nvme.value = os.path.join(
+                self.server_managers[0].get_config_value("path"), "daos_control", "engine0",
+                "daos_nvme.conf")
+
+        self.log_step("Perform dmg system stop to run dlck command")
+        dmg.system_stop()
+
+        if self.server_managers[0].manager.job.using_control_metadata:
+            dlck.nvme.value = os.path.join(
+                self.server_managers[0].get_config_value("path"), "daos_control", "engine0",
+                "daos_nvme.conf")
+            # Mount 128G of memory-backed tmpfs for testing purposes
+            cmd = 'sudo "mkdir -p /mnt/dlck_test && ' \
+                  'mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test"'
+            run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd)
+            # Display the tmpfs mount
+            run_remote(log=self.log, hosts=self.hostlist_servers, command="mount")
+            run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h")
+            dlck.storage.value = "/mnt/dlck_test"
+
+        self.log_step("Run dlck command to check the health of the pool and storage")
+        result = dlck.run()
+
+        self.log_step("Perform dmg system start after running dlck command")
+        dmg.system_start()
+        if not result.passed:
+            self.fail(f"dlck failed on {result.failed_hosts}")
+        self.log.info("dlck basic test passed with no errors")
+
+    def test_dlck_basic_fault(self):
+        """Basic Fault Test: Run 'dlck' injecting basic faults.
+
+        :avocado: tags=all,full_regression
+        :avocado: tags=hw,medium
+        :avocado: tags=recovery,dlck_cmd,faults
+        :avocado: tags=DlckBasicTest,test_dlck_basic_fault
+        """
+        errors = []
+        faults_object = FaultInjection()
+        faults_dict = faults_object.get_faults_dict()
+        fault_list = self.params.get("fault_list", '/run/dlck_test_faults/*')
+        self.log.info("Test log dir %s", self.test_env.log_dir)
+        fault_inject_file = os.path.join(self.test_env.log_dir, "fi.yaml")
+        self.log.info("Fault injection file: %s", fault_inject_file)
+        self.log.info("Faults: %s", fault_list)
+        self.log.info("Faults dict: %s", faults_dict)
+        dmg = self.get_dmg_command()
+        pool = self.get_pool()
+        dlck = self.get_dlck_command()
+        dlck.pool_uuid.value = pool.uuid
+        dlck.log_dir.value = self.test_env.log_dir
+        dlck.exit_status_exception = False
+        dlck.env["D_FI_CONFIG"] = fault_inject_file
+        dlck.storage.value = self.server_managers[0].get_config_value("scm_mount")
+
+        self.log_step("Perform dmg system stop to run dlck command")
+        dmg.system_stop()
+
+        if self.server_managers[0].manager.job.using_control_metadata:
+            dlck.nvme.value = os.path.join(
+                self.server_managers[0].get_config_value("path"), "daos_control", "engine0",
+                "daos_nvme.conf")
+            # Mount 128G of memory-backed tmpfs for testing purposes
+            cmd = 'sudo "mkdir -p /mnt/dlck_test && ' \
+                  'mount -t tmpfs -o size=128G tmpfs /mnt/dlck_test"'
+            run_remote(log=self.log, hosts=self.hostlist_servers, command=cmd)
+            # Display the tmpfs mount
+            run_remote(log=self.log, hosts=self.hostlist_servers, command="mount")
+            run_remote(log=self.log, hosts=self.hostlist_servers, command="df -h")
+            dlck.storage.value = "/mnt/dlck_test"
+
+        for test_fault in fault_list:
+            if test_fault == "DAOS_FAULT_POOL_OPEN_BIO" and \
+               not self.server_managers[0].manager.job.using_control_metadata:
+                self.log.info("Skipping test fault %s as control metadata is not used", test_fault)
+                continue
+            fault_config = {
+                'fault_config': [
+                    {key: str(value) for key, value in faults_dict[test_fault].items()}]}
+            with open(fault_inject_file, 'w') as f:
+                yaml.dump(fault_config, f)
+            self.log.info("Reading the updated fault injection file contents")
+
+            with open(fault_inject_file, 'r') as f:
+                file_data = f.read()
+                self.log.info("\n %s", file_data)
+            distribute_files(self.log, self.hostlist_servers, fault_inject_file,
+                             fault_inject_file)
+            self.log_step("Run dlck command after injecting fault")
+            result = dlck.run()
+            error = self.check_dlck_result(result, test_fault)
+            if error:
+                errors.append(error)
+
+        self.log_step("Perform dmg system start after running dlck command")
+        dmg.system_start()
+
+        if errors:
+            self.fail(f"dlck basic test failed with errors: {errors}")
+        self.log.info("dlck basic fault test passed with no errors")

src/tests/ftest/recovery/dlck_basic.yaml

@@ -0,0 +1,37 @@
+hosts:
+  test_servers: 1
+  test_clients: 1
+
+timeout: 420
+server_config:
+  name: daos_server
+  engines_per_host: 1
+  engines:
+    0:
+      targets: 4
+      storage: auto
+
+pool:
+  size: 30%
+
+dlck_test_faults:
+  fault_list:
+    - DLCK_FAULT_GETGRNAM
+    - DLCK_MOCK_NO_DAOS_SERVER_GROUP
+    - DLCK_FAULT_CREATE_POOL_DIR
+    - DLCK_FAULT_CREATE_LOG_DIR
+    - DLCK_FAULT_ENGINE_START
+    - DAOS_FAULT_POOL_NVME_HEALTH
+    - DAOS_FAULT_POOL_OPEN_UMEM
+    - DAOS_FAULT_POOL_OPEN_MAGIC
+    - DAOS_FAULT_POOL_OPEN_VERSION
+    - DAOS_FAULT_POOL_OPEN_UUID
+    - DAOS_FAULT_BTREE_OPEN_INV_CLASS_28
+    - DAOS_FAULT_BTREE_OPEN_INV_CLASS_29
+    - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_28
+    - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS_29
+    - DAOS_FAULT_BTREE_FEATURES_28
+    - DAOS_FAULT_BTREE_FEATURES_29
+    - DAOS_FAULT_POOL_EXT_PADDING
+    - DAOS_FAULT_POOL_EXT_RESERVED
+    - DAOS_FAULT_POOL_OPEN_BIO

src/tests/ftest/util/collection_utils.py

@@ -1,6 +1,6 @@
 """
   (C) Copyright 2022-2024 Intel Corporation.
-  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+  (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -30,7 +30,8 @@
 FAILURE_TRIGGER = "00_trigger-launch-failure_00"
 TEST_EXPECT_CORE_FILES = ["./harness/core_files.py"]
 TEST_RESULTS_DIRS = (
-    "daos_configs", "daos_logs", "cart_logs", "daos_dumps", "valgrind_logs", "stacktraces")
+    "daos_configs", "daos_logs", "cart_logs", "daos_dumps", "valgrind_logs", "stacktraces",
+    "dlck_check")
 
 
 def stop_daos_agent_services(logger, test):
@@ -225,7 +226,7 @@ def check_server_storage(logger, test, test_result, stage):
 
 
 def archive_files(logger, summary, hosts, source, pattern, destination, depth, threshold, timeout,
-                  test_result, test=None):
+                  test_result, test=None, file_type="f"):
     # pylint: disable=too-many-arguments
     """Archive the files from the source to the destination.
 
@@ -241,6 +242,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t
         timeout (int): number of seconds to wait for the command to complete.
         test_result (TestResult): the test result used to update the status of the test
         test (TestInfo, optional): the test information. Defaults to None.
+        file_type (str, optional): type of file to search for. Defaults to "f".
 
     Returns:
         int: status code: 0 = success, 16 = failure
@@ -255,7 +257,8 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t
     logger.debug("  Local host:   %s", hosts.intersection(get_local_host()))
 
     # List any remote files and their sizes and determine which hosts contain these files
-    return_code, file_hosts = list_files(logger, hosts, source, pattern, depth, test_result)
+    return_code, file_hosts = list_files(logger, hosts, source, pattern, depth,
+                                         test_result, file_type)
     if not file_hosts:
         # If no files are found then there is nothing else to do
         logger.debug("No %s files found on %s", os.path.join(source, pattern), hosts)
@@ -281,7 +284,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t
 
     # Move the test files to the test-results directory on this host
     return_code |= move_files(
-        logger, file_hosts, source, pattern, destination, depth, timeout, test_result)
+        logger, file_hosts, source, pattern, destination, depth, timeout, test_result, file_type)
 
     if test and "core files" in summary:
         # Process the core files
@@ -290,16 +293,17 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t
     return return_code
 
 
-def list_files(logger, hosts, source, pattern, depth, test_result):
+def list_files(logger, hosts, source, pattern, depth, test_result, file_type="f"):
     """List the files in source with that match the pattern.
 
     Args:
         logger (Logger): logger for the messages produced by this method
-        hosts (NodSet): hosts on which the files are located
+        hosts (NodeSet): hosts on which the files are located
         source (str): where the files are currently located
         pattern (str): pattern used to limit which files are processed
         depth (int): max depth for find command
         test_result (TestResult): the test result used to update the status of the test
+        file_type (str, optional): type of file to search for. Defaults to "f".
 
     Returns:
         tuple: a tuple containing:
@@ -313,15 +317,16 @@ def list_files(logger, hosts, source, pattern, depth, test_result):
     logger.debug("-" * 80)
     logger.debug("Listing any %s files on %s", source_files, hosts)
     other = ["-printf", "'%M %n %-12u %-12g %12k %t %p\n'"]
-    result = run_remote(logger, hosts, find_command(source, pattern, depth, other))
+    result = run_remote(logger, hosts, find_command(source, pattern, depth,
+                                                    other, file_type))
     if not result.passed:
         message = f"Error determining if {source_files} files exist on {result.failed_hosts}"
         test_result.fail_test(logger, "Process", message)
         status = 16
     else:
         for data in result.output:
             for line in data.stdout:
-                if source in line:
+                if (source in line) or re.search(source, line):
                     logger.debug("Found at least one file match on %s: %s", data.hosts, line)
                     hosts_with_files.add(data.hosts)
                     break
@@ -466,18 +471,20 @@ def compress_files(logger, hosts, source, pattern, depth, test_result):
     return 0
 
 
-def move_files(logger, hosts, source, pattern, destination, depth, timeout, test_result):
+def move_files(logger, hosts, source, pattern, destination, depth,
+               timeout, test_result, file_type="f"):
     """Move files from the source to the destination.
 
     Args:
         logger (Logger): logger for the messages produced by this method
-        hosts (NodSet): hosts on which the files are located
+        hosts (NodeSet): hosts on which the files are located
         source (str): where the files are currently located
         pattern (str): pattern used to limit which files are processed
         destination (str): where the files should be moved to on this host
         depth (int): max depth for find command
         timeout (int): number of seconds to wait for the command to complete.
         test_result (TestResult): the test result used to update the status of the test
+        file_type (str, optional): type of file to search for. Defaults to "f".
 
     Returns:
         int: status code: 0 = success, 16 = failure
@@ -491,7 +498,8 @@ def move_files(logger, hosts, source, pattern, destination, depth, timeout, test
     if "stacktrace" in destination or "daos_dumps" in destination:
         # pylint: disable=import-outside-toplevel
         other = ["-print0", "|", "xargs", "-0", "-r0", "sudo", "-n", get_chown_command()]
-        result = run_remote(logger, hosts, find_command(source, pattern, depth, other))
+        result = run_remote(logger, hosts, find_command(source, pattern, depth,
+                                                        other, file_type))
         if not result.passed:
             message = (f"Error changing {os.path.join(source, pattern)} file permissions on "
                        f"{result.failed_hosts}")
@@ -528,8 +536,8 @@ def move_files(logger, hosts, source, pattern, destination, depth, timeout, test
         return return_code
 
     # Move all the source files matching the pattern into the temporary remote directory
-    other = f"-print0 | xargs -0 -r0 -I '{{}}' {sudo_command}mv '{{}}' '{tmp_copy_dir}'/"
-    result = run_remote(logger, hosts, find_command(source, pattern, depth, other))
+    other = f"-print0 | xargs -0 -r0 -I '{{}}' {sudo_command} mv '{{}}' '{tmp_copy_dir}'/"
+    result = run_remote(logger, hosts, find_command(source, pattern, depth, other, file_type))
     if not result.passed:
         message = (f"Error moving files to temporary remote copy directory '{tmp_copy_dir}' on "
                    f"{result.failed_hosts}")
@@ -997,6 +1005,15 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a
     # size exceeding the threshold.
     test_env = TestEnvironment()
     if archive:
+        logger.debug("Making all '%s' files accessible for archiving", test_env.log_dir)
+        result = run_remote(logger, test.host_info.all_hosts,
+                            "sudo -n {}".format(get_chown_command(options="-R",
+                                                                  file=test_env.log_dir)))
+        if not result.passed:
+            message = f"Error making {test_env.log_dir} files accessible for archiving"
+            test_result.fail_test(logger, "Process", message)
+            return_code = 16
+
         remote_files = OrderedDict()
         remote_files["local configuration files"] = {
             "source": test_env.log_dir,
@@ -1047,6 +1064,15 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a
             "depth": 1,
             "timeout": 900,
         }
+        remote_files["dlck files"] = {
+            "source": test_env.log_dir,
+            "destination": os.path.join(job_results_dir, "latest", TEST_RESULTS_DIRS[6]),
+            "pattern": "dlck_check_*",
+            "hosts": test.host_info.all_hosts,
+            "depth": 1,
+            "file_type": "d",
+            "timeout": 900,
+        }
         for index, hosts in enumerate(core_files):
             remote_files[f"core files {index + 1}/{len(core_files)}"] = {
                 "source": core_files[hosts]["path"],
@@ -1062,7 +1088,7 @@ def collect_test_result(logger, test, test_result, job_results_dir, stop_daos, a
             return_code |= archive_files(
                 logger, summary, data["hosts"].copy(), data["source"], data["pattern"],
                 data["destination"], data["depth"], threshold, data["timeout"],
-                test_result, test)
+                test_result, test, file_type=data.get("file_type", "f"))
 
     # Generate a steps.log file
     return_code |= create_steps_log(logger, job_results_dir, test_result)