From 22f74c8812fa973418d96076e6b6d67da086be45 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 01/17] DAOS-15603: Support of FIO with ansible ftest script

Miscellaneous improvements allowing to run fio functional tests, and to
use the shared cluster.

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/README.md                 |  13 ++-
 utils/ansible/ftest/file/configure-nofile.sh  |  20 ++++
 utils/ansible/ftest/file/enable-coredumps.sh  |  44 ++++++++
 utils/ansible/ftest/inventory-sample.yml      |   4 +-
 utils/ansible/ftest/tasks/daos_base.yml       |  29 +++--
 utils/ansible/ftest/tasks/daos_clients.yml    |   9 ++
 utils/ansible/ftest/tasks/daos_dev.yml        |  11 +-
 utils/ansible/ftest/tasks/daos_servers.yml    |   2 +-
 .../ftest/tasks/el8/clients_dependencies.yml  |   8 ++
 .../ftest/tasks/el8/devs_dependencies.yml     |  16 ---
 .../ftest/tasks/el8/servers_dependencies.yml  |   8 ++
 .../ansible/ftest/templates/daos-launch.sh.j2 |  30 ++---
 .../ftest/templates/daos-launch_nlt.sh.j2     |  26 +++--
 utils/ansible/ftest/templates/daos-make.sh.j2 | 106 +++++++++++++-----
 utils/ansible/ftest/vars/Rocky8.yml           |  54 ++++++---
 utils/ansible/ftest/vars/defaults.yml         |   7 +-
 16 files changed, 283 insertions(+), 104 deletions(-)
 create mode 100644 utils/ansible/ftest/file/configure-nofile.sh
 create mode 100644 utils/ansible/ftest/file/enable-coredumps.sh

diff --git a/utils/ansible/ftest/README.md b/utils/ansible/ftest/README.md
index d01e21e61ea..5bc4ab16f04 100644
--- a/utils/ansible/ftest/README.md
+++ b/utils/ansible/ftest/README.md
@@ -67,15 +67,17 @@ The inventory should also contain a set of mandatory and optional variables.
   sub-cluster.
 - **daos\_source\_dir**: mandatory variable only used by the node of the *daos\_dev* group defining
   the path of the directory containing the DAOS source code.
+- **daos\_ofi\_provider**: optional variable (default value: ofi+tcp;ofi_rxm) defining the network
+  provider to be used by the DAOS engines.  It also defines which network rpms driver to install
+  (e.g. ib mellanox driver).
 - **daos\_ofi\_interface**: optional variable only used by the node of the daos\_dev group defining
   the network interface to use.  When this variable is not defined, the network interface is
   arbitrarily selected by DAOS.
 - **daos\_hugepages\_nb**: optional variable (default value: 4096) only used by the nodes of the
-  *daos\_servers* group defining the number of hugepages to be allocated by the linux kernel.
-- **daos\_avocado\_version**: optional variable (default value: "2.4.3") only used by the node of
-  the *daos\_dev* group defining the version of *avocado* to install.
-- **daos\_avocado\_framework\_version**: optional variable (default value: "82.1") only used by the
-  node of the *daos\_dev* group defining the version of *avocado\_framework* to install.
+  *daos\_servers* group.  This variable defines the number of hugepages to be allocated by the linux
+  kernel.
+- **daos\_build\_mode**: optional variable (default value: DEBUG) only used by the nodes of the
+  *daos\_dev* group.  This variable defines in which mode DAOS shall be compiled.
 
 Different file format (e.g. YAML, INI, etc.) and file tree structure are supported to define an
 ansible inventory.  The following simple ansible inventory describe for example in one YAML file
@@ -90,6 +92,7 @@ all:
       vars:
         daos_source_dir: /home/foo/work/daos
         daos_ofi_interface: eth0
+        daos_build_mode: DEBUG
       hosts:
         wolf-666:
     daos_servers:
diff --git a/utils/ansible/ftest/file/configure-nofile.sh b/utils/ansible/ftest/file/configure-nofile.sh
new file mode 100644
index 00000000000..1ca680d9350
--- /dev/null
+++ b/utils/ansible/ftest/file/configure-nofile.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# set -x
+set -u -e -o pipefail
+
+if [[ $(id -u) -ne 0 ]] ; then
+	echo "[ERROR] Could only be used by root"
+fi
+
+sed -i -E -e "/DefaultLimitNOFILE/s/.*/DefaultLimitNOFILE=infinity:infinity/" /etc/systemd/system.conf
+
+cat > /etc/security/limits.d/daos-nofile.conf << EOF
+# /etc/security/limits.d/daos-nofile.conf
+
+# <domain>	<type>	<item>	<value>
+*		hard	nofile	unlimited
+*		soft	nofile	unlimited
+EOF
+
+systemctl daemon-reexec
diff --git a/utils/ansible/ftest/file/enable-coredumps.sh b/utils/ansible/ftest/file/enable-coredumps.sh
new file mode 100644
index 00000000000..6746ea718b2
--- /dev/null
+++ b/utils/ansible/ftest/file/enable-coredumps.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# set -x
+set -u -e -o pipefail
+
+if [[ $(id -u) -ne 0 ]] ; then
+	echo "[ERROR] Could only be used by root"
+fi
+
+sed -i -E -e "/DefaultLimitCORE/s/.*/DefaultLimitCORE=infinity:infinity/" /etc/systemd/system.conf
+
+cat > /etc/security/limits.d/daos-coredumps.conf << EOF
+# /etc/security/limits.d/daos-coredumps.conf
+
+# <domain>	<type>	<item>	<value>
+*		hard	core	unlimited
+*		soft	core	unlimited
+EOF
+
+cat > /etc/sysctl.d/daos-coredumps.conf << EOF
+# /etc/sysctl.d/daos-coredumps.conf
+
+fs.suid_dumpable=1
+EOF
+sysctl -q -w "fs.suid_dumpable=1"
+sysctl -p
+
+cat > /etc/profile.d/daos-coredumps.sh  << EOF
+# Enable core dump
+ulimit -c unlimited
+EOF
+
+mkdir -p /etc/systemd/coredump.conf.d/
+cat > /etc/systemd/coredump.conf.d/daos-coredumps.conf << EOF
+# /etc/systemd/coredump.conf.d/daos-coredumps.conf
+
+[Coredump]
+Storage=external
+Compress=yes
+ProcessSizeMax=8G
+ExternalSizeMax=8G
+KeepFree=2G
+EOF
+systemctl daemon-reexec
diff --git a/utils/ansible/ftest/inventory-sample.yml b/utils/ansible/ftest/inventory-sample.yml
index 1ed11b68a8b..e81d8376268 100644
--- a/utils/ansible/ftest/inventory-sample.yml
+++ b/utils/ansible/ftest/inventory-sample.yml
@@ -3,11 +3,13 @@
 all:
   vars:
     daos_runtime_dir: /home/foo/daos
+    # daos_ofi_provider: ofi+tcp;ofi_rxm
   children:
     daos_dev:
       vars:
         daos_source_dir: /home/foo/work/daos
-        daos_ofi_interface: eth0
+        # daos_ofi_interface: eth0
+        # daos_build_mode: DEBUG
       hosts:
         wolf-666:
     daos_servers:
diff --git a/utils/ansible/ftest/tasks/daos_base.yml b/utils/ansible/ftest/tasks/daos_base.yml
index 6ac6ac8f7d1..da555c5d19d 100644
--- a/utils/ansible/ftest/tasks/daos_base.yml
+++ b/utils/ansible/ftest/tasks/daos_base.yml
@@ -37,13 +37,28 @@
     mode: '0755'
     recurse: true
 
-- name: Create MPI module configuration file
+- name: Copy bash script enable-coredumps.sh
   remote_user: root
-  file:
-    src: "{{ daos_openmpi_module_path }}"
-    path: /etc/modulefiles/mpi/mlnx_openmpi-x86_64
-    state: link
+  copy:
+    src: file/enable-coredumps.sh
+    dest: /root/enable-coredumps.sh
     owner: root
     group: root
-    mode: '0755'
-    force: true
+    mode: '0644'
+
+- name: Enable CORE dump
+  remote_user: root
+  command: /bin/bash /root/enable-coredumps.sh
+
+- name: Copy bash script configure-nofile.sh
+  remote_user: root
+  copy:
+    src: file/configure-nofile.sh
+    dest: /root/configure-nofile.sh
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Configure Number of Open Files
+  remote_user: root
+  command: /bin/bash /root/configure-nofile.sh
diff --git a/utils/ansible/ftest/tasks/daos_clients.yml b/utils/ansible/ftest/tasks/daos_clients.yml
index 786720283ed..5b5116a04ff 100644
--- a/utils/ansible/ftest/tasks/daos_clients.yml
+++ b/utils/ansible/ftest/tasks/daos_clients.yml
@@ -69,3 +69,12 @@
     group: root
     mode: '0644'
     backup: true
+
+- name: Update search path commands
+  remote_user: root
+  lineinfile:
+    path: /etc/security/pam_env.conf
+    state: present
+    regexp: '^PATH='
+    line: "PATH={{ daos_runtime_dir }}/install/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/bin/syscfg:/root/bin"
+    backup: true
diff --git a/utils/ansible/ftest/tasks/daos_dev.yml b/utils/ansible/ftest/tasks/daos_dev.yml
index 9b604181e98..e54a7f1a911 100644
--- a/utils/ansible/ftest/tasks/daos_dev.yml
+++ b/utils/ansible/ftest/tasks/daos_dev.yml
@@ -44,17 +44,16 @@
     group: daos_server
     mode: 0700
 
-- name: Install dependencies packages
-  include_tasks: "{{ item }}"
-  with_first_found:
-    - "{{ ansible_distribution | replace(' ', '_') }}{{ ansible_distribution_major_version}}/devs_dependencies.yml"
-
 - name: Create DAOS runtime directory
   file:
     path: "{{ daos_runtime_dir }}"
     state: directory
     mode: '0755'
-    recurse: true
+
+- name: Install dependencies packages
+  include_tasks: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_distribution | replace(' ', '_') }}{{ ansible_distribution_major_version}}/devs_dependencies.yml"
 
 - name: Install DAOS make helper script
   template:
diff --git a/utils/ansible/ftest/tasks/daos_servers.yml b/utils/ansible/ftest/tasks/daos_servers.yml
index faf4c6f3680..3ea6973e13e 100644
--- a/utils/ansible/ftest/tasks/daos_servers.yml
+++ b/utils/ansible/ftest/tasks/daos_servers.yml
@@ -19,7 +19,7 @@
 - name: Check Huge Pages
   remote_user: root
   daos_hugepages:
-    size: "{{ daos_hugepages_nb | default(4096) }}"
+    size: "{{ daos_hugepages_nb | default(daos_hugepages_nb_default) }}"
 
 - name: Add DAOS server groups
   remote_user: root
diff --git a/utils/ansible/ftest/tasks/el8/clients_dependencies.yml b/utils/ansible/ftest/tasks/el8/clients_dependencies.yml
index 5a801e15bbd..080290f4a1d 100644
--- a/utils/ansible/ftest/tasks/el8/clients_dependencies.yml
+++ b/utils/ansible/ftest/tasks/el8/clients_dependencies.yml
@@ -6,3 +6,11 @@
     state: latest
     update_cache: true
     name: "{{ daos_base_deps.rpms + daos_client_deps.rpms }}"
+
+- name: Install MOFED rpms
+  remote_user: root
+  dnf:
+    state: latest
+    update_cache: true
+    name: "{{ mlnx_deps.rpms }}"
+  when: daos_ofi_provider | default(daos_ofi_provider_default) == "ofi+verbs;ofi_rxm"
diff --git a/utils/ansible/ftest/tasks/el8/devs_dependencies.yml b/utils/ansible/ftest/tasks/el8/devs_dependencies.yml
index 9510bed9b52..f3501b900b7 100644
--- a/utils/ansible/ftest/tasks/el8/devs_dependencies.yml
+++ b/utils/ansible/ftest/tasks/el8/devs_dependencies.yml
@@ -6,19 +6,3 @@
     state: latest
     update_cache: true
     name: "{{ daos_base_deps.rpms + daos_dev_deps.rpms }}"
-
-- name: Install DAOS building dependencies
-  remote_user: root
-  command:
-    dnf builddep --nobest --assumeyes "{{ playbook_dir }}/file/daos.spec"
-  args:
-    warn: false
-  register: dnf_result
-  become: true
-  changed_when: '"Nothing to do" not in dnf_result.stdout'
-
-- name: Install python development dependencies
-  remote_user: root
-  pip:
-    state: present
-    name: "{{ daos_dev_deps.pips }}"
diff --git a/utils/ansible/ftest/tasks/el8/servers_dependencies.yml b/utils/ansible/ftest/tasks/el8/servers_dependencies.yml
index 5ef0d5d3af8..1d7f8f28fc3 100644
--- a/utils/ansible/ftest/tasks/el8/servers_dependencies.yml
+++ b/utils/ansible/ftest/tasks/el8/servers_dependencies.yml
@@ -6,3 +6,11 @@
     state: latest
     update_cache: true
     name: "{{ daos_base_deps.rpms + daos_server_deps.rpms }}"
+
+- name: Install MOFED rpms
+  remote_user: root
+  dnf:
+    state: latest
+    update_cache: true
+    name: "{{ mlnx_deps.rpms }}"
+  when: daos_ofi_provider | default(daos_ofi_provider_default) == "ofi+verbs;ofi_rxm"
diff --git a/utils/ansible/ftest/templates/daos-launch.sh.j2 b/utils/ansible/ftest/templates/daos-launch.sh.j2
index 56c1f65df04..d008beba1e2 100644
--- a/utils/ansible/ftest/templates/daos-launch.sh.j2
+++ b/utils/ansible/ftest/templates/daos-launch.sh.j2
@@ -1,10 +1,11 @@
 #!/bin/bash
 # shellcheck disable=all # Jinja2 template can not be parsed with shellcheck
 
-set -o pipefail
+set -u -o pipefail
 
-VERSION=0.1
-CWD="$(realpath "$(dirname $0)")"
+VERSION=0.2
+CWD="$(realpath "${0%}")"
+CWD="${CWD%/*}"
 
 DAOS_INSTALL_DIR="{{ daos_runtime_dir }}/install"
 DAOS_SOURCE_DIR="{{ daos_source_dir }}"
@@ -43,6 +44,9 @@ TRACE_LEVEL_VERBOSE=1
 TRACE_LEVEL_DEBUG=2
 TRACE_LEVEL=$TRACE_LEVEL_STANDARD
 
+START_CLEANUP="false"
+END_CLEANUP="false"
+
 function debug
 {
 	if [[ $TRACE_LEVEL -ge $TRACE_LEVEL_DEBUG ]]
@@ -194,7 +198,7 @@ function cleanup
 opts=()
 args=()
 for item in "$@" ; do
-	if [[ ${#args} -eq 0 && "$item" != "--" ]] ; then
+	if [[ ${#args[@]} -eq 0 && "$item" != "--" ]] ; then
 		[[ ${item::1} == "-" ]] || fatal "unrecognized option \"$item\""
 		opts+=("$item")
 	else
@@ -209,13 +213,13 @@ eval set -- "$OPTIONS"
 while true
 do
 	case "$1" in
-		-s|--start-cleanup) START_CLEANUP=true ; shift 1;;
-		-e|--end-cleanup) END_CLEANUP=true ; shift 1;;
+		-s|--start-cleanup) START_CLEANUP="true" ; shift 1;;
+		-e|--end-cleanup) END_CLEANUP="true" ; shift 1;;
 		-h|--help) usage ; exit 0;;
 		-V|--version) echo "daos-launch.sh version=$VERSION" ; exit 0 ;;
-		-v|--verbose) TRACE_LEVEL=$TRACE_LEVEL_VERBOSE ; shift 1 ;;
-		-D|--debug) TRACE_LEVEL=$TRACE_LEVEL_DEBUG ; set -x ; shift 1 ;;
-		-q|--quiet) TRACE_LEVEL=$TRACE_LEVEL_QUIET ; shift 1 ;;
+		-v|--verbose) TRACE_LEVEL="$TRACE_LEVEL_VERBOSE" ; shift 1 ;;
+		-D|--debug) TRACE_LEVEL="$TRACE_LEVEL_DEBUG" ; set -x ; shift 1 ;;
+		-q|--quiet) TRACE_LEVEL="$TRACE_LEVEL_QUIET" ; shift 1 ;;
 		--) shift ; break ;;
 		*) fatal "unrecognized option" ;;
 	esac
@@ -223,7 +227,7 @@ done
 
 check_cmds $PYTHON_EXE $CLUSH_EXE $MODULE_EXE
 
-if [[ -n $START_CLEANUP ]] ; then
+if "$START_CLEANUP" ; then
 	cleanup
 fi
 
@@ -238,9 +242,9 @@ fi
 info "Parametrization of the simulation"
 cd "$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
-common_opts="-ar --test_servers=$SERVERS_LIST --test_clients=$CLIENTS_LIST"
+common_opts="-ar --provider={{ daos_ofi_provider | default(daos_ofi_provider_default) }} --test_servers=$SERVERS_LIST --test_clients=$CLIENTS_LIST"
 {% else %}
-common_opts="-ar --test_servers=$SERVERS_LIST"
+common_opts="-ar --provider={{ daos_ofi_provider | default(daos_ofi_provider_default) }} --test_servers=$SERVERS_LIST"
 {% endif %}
 
 info "Starting functional test simulation"
@@ -251,7 +255,7 @@ $PYTHON_EXE launch.py $common_opts "${args[@]}"
 {% endif %}
 rc=$?
 
-if [[ -n $END_CLEANUP ]] ; then
+if "$END_CLEANUP" ; then
 	cleanup
 fi
 
diff --git a/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2 b/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
index c4009a0f2a5..548d81009b8 100644
--- a/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
+++ b/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
@@ -1,10 +1,11 @@
 #!/bin/bash
 # shellcheck disable=all # Jinja2 template can not be parsed with shellcheck
 
-set -o pipefail
+set -u -o pipefail
 
-VERSION=0.1
-CWD="$(realpath "$(dirname $0)")"
+VERSION=0.2
+CWD="$(realpath "${0%}")"
+CWD="${CWD%/*}"
 
 DAOS_INSTALL_DIR="{{ daos_runtime_dir }}/install"
 DAOS_SOURCE_DIR="{{ daos_source_dir }}"
@@ -38,6 +39,9 @@ TRACE_LEVEL_VERBOSE=1
 TRACE_LEVEL_DEBUG=2
 TRACE_LEVEL=$TRACE_LEVEL_STANDARD
 
+START_CLEANUP="false"
+END_CLEANUP="false"
+
 function debug
 {
 	if [[ $TRACE_LEVEL -ge $TRACE_LEVEL_DEBUG ]]
@@ -155,7 +159,7 @@ function cleanup
 opts=()
 args=()
 for item in "$@" ; do
-	if [[ ${#args} -eq 0 && "$item" != "--" ]] ; then
+	if [[ ${#args[@]} -eq 0 && "$item" != "--" ]] ; then
 		[[ ${item::1} == "-" ]] || fatal "unrecognized option \"$item\""
 		opts+=("$item")
 	else
@@ -170,13 +174,13 @@ eval set -- "$OPTIONS"
 while true
 do
 	case "$1" in
-		-s|--start-cleanup) START_CLEANUP=true ; shift 1;;
-		-e|--end-cleanup) END_CLEANUP=true ; shift 1;;
+		-s|--start-cleanup) START_CLEANUP="true" ; shift 1;;
+		-e|--end-cleanup) END_CLEANUP="true" ; shift 1;;
 		-h|--help) usage ; exit 0;;
 		-V|--version) echo "daos-launch.sh version=$VERSION" ; exit 0 ;;
-		-v|--verbose) TRACE_LEVEL=$TRACE_LEVEL_VERBOSE ; shift 1 ;;
-		-D|--debug) TRACE_LEVEL=$TRACE_LEVEL_DEBUG ; set -x ; shift 1 ;;
-		-q|--quiet) TRACE_LEVEL=$TRACE_LEVEL_QUIET ; shift 1 ;;
+		-v|--verbose) TRACE_LEVEL="$TRACE_LEVEL_VERBOSE" ; shift 1 ;;
+		-D|--debug) TRACE_LEVEL="$TRACE_LEVEL_DEBUG" ; set -x ; shift 1 ;;
+		-q|--quiet) TRACE_LEVEL="$TRACE_LEVEL_QUIET" ; shift 1 ;;
 		--) shift ; break ;;
 		*) fatal "unrecognized option" ;;
 	esac
@@ -184,7 +188,7 @@ done
 
 check_cmds $PYTHON_EXE $CLUSH_EXE $MODULE_EXE
 
-if [[ -n $START_CLEANUP ]] ; then
+if "$START_CLEANUP" ; then
 	cleanup
 
 	info "Cleaning NLT logs"
@@ -209,7 +213,7 @@ $PYTHON_EXE launch.py $common_opts "${args[@]}"
 rc=$?
 popd > /dev/null
 
-if [[ -n $END_CLEANUP ]] ; then
+if "$END_CLEANUP" ; then
 	cleanup
 fi
 
diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index 4ad23399ed5..34ac736b537 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -1,10 +1,11 @@
 #!/bin/bash
 # shellcheck disable=all # Jinja2 template can not be parsed with shellcheck
 
-set -o pipefail
+set -u -o pipefail
 
-VERSION=0.3
-CWD="$(realpath "$(dirname $0)")"
+VERSION=0.4
+CWD="$(realpath "${0%}")"
+CWD="${CWD%/*}"
 declare -A VALID_CMDS=([install]=true [update]=true)
 
 DAOS_INSTALL_DIR="{{ daos_runtime_dir }}/install"
@@ -13,13 +14,18 @@ SERVERS_LIST="{{ groups["daos_servers"] | join(',') }}"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
 CLIENTS_LIST="{{ groups["daos_clients"] | join(',') }}"
 {% endif %}
+BUILD_TYPE="{{ daos_build_mode | default(daos_build_mode_default) }}"
 
 SCONS_EXE=scons-3
+SSH_EXE=ssh
 CLUSH_EXE=clush
+NODESET_EXE=nodeset
 MODULE_EXE=module
 CMAKE_EXE=cmake3
 MAKE_EXE=make
 WGET_EXE=wget
+PYTHON_EXE=python3.11
+PIP_EXE=pip
 
 CLUSH_OPTS="-bL -S"
 
@@ -52,6 +58,7 @@ TRACE_LEVEL_DEBUG=2
 TRACE_LEVEL=$TRACE_LEVEL_STANDARD
 
 JOBS_NB=1
+FORCE_INSTALL="false"
 
 function debug
 {
@@ -97,6 +104,7 @@ function fatal
 
 function check_cmds
 {
+	hash -r
 	for cmd in $@
 	do
 		{ hash $cmd > "/dev/null" 2>&1 ; } || { fatal "$cmd command not installed" ; }
@@ -116,6 +124,7 @@ function usage
 		       --vol-daos <path>	Path of vol-daos source tree
 		       --mpifileutils <path>    Path of mpi file utils source tree
 		       --ior <path>             Path of ior file utils source tree
+		       --fio <path>             Path of fio file utils source tree
 		   -j, --jobs <job>             Allow N jobs at once; one job with no arg
 		   -f, --force                  Overwrite the current install of DAOS
 		   -h, --help                   Show this help message and exit
@@ -140,7 +149,7 @@ function run
 	fi
 }
 
-OPTIONS=$(getopt -o "j:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,jobs:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
+OPTIONS=$(getopt -o "j:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,fio:,jobs:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
 eval set -- "$OPTIONS"
 while true
 do
@@ -150,8 +159,9 @@ do
 		--vol-daos) VOL_DAOS_PATH="$2" ; shift 2 ;;
 		--mpifileutils) MPI_FILE_UTILS_PATH="$2" ; shift 2 ;;
 		--ior) IOR_PATH="$2" ; shift 2 ;;
+		--fio) FIO_PATH="$2" ; shift 2 ;;
 		-j|--jobs) JOBS_NB="$2" ; shift 2 ;;
-		-f|--force) FORCE_INSTALL=true ; shift 1;;
+		-f|--force) FORCE_INSTALL="true" ; shift 1;;
 		-h|--help) usage ; exit 0;;
 		-V|--version) echo "daos-make.sh version=$VERSION" ; exit 0 ;;
 		-v|--verbose) TRACE_LEVEL=$TRACE_LEVEL_VERBOSE ; shift 1 ;;
@@ -162,33 +172,37 @@ do
 	esac
 done
 
-[[ $1 ]] || fatal "Command not defined: use install or update"
+[[ ${1:+x} ]] || fatal "Command not defined: use install or update"
 CMD="$1"
-[[ ${VALID_CMDS[$CMD]} == true ]] || fatal "Invalid command \"$CMD\": use install or update"
+${VALID_CMDS[$CMD]} || fatal "Invalid command \"$CMD\": use install or update"
 
-check_cmds $SCONS_EXE $MODULE_EXE $CLUSH_EXE $WGET_EXE $MAKE_EXE $CMAKE_EXE
+check_cmds $SSH_EXE $CLUSH_EXE $NODESET_EXE
 
 set -e
 
-info "Mount of home partition"
+info "Checking of home partition"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
-run $CLUSH_EXE $CLUSH_OPTS -l root -w $CLIENTS_LIST -w $SERVERS_LIST mountpoint --quiet "$HOME"
+for host in $($NODESET_EXE -e $CLIENTS_LIST $SERVERS_LIST)
 {% else %}
-run $CLUSH_EXE $CLUSH_OPTS -w $SERVERS_LIST mountpoint --quiet "$HOME"
+for host in $($NODESET_EXE -e $SERVERS_LIST)
 {% endif %}
+do
+	run $SSH_EXE root@$host mountpoint --quiet "$HOME" || fatal "Home directory '$home' not mounted on $host"
+done
 
 for dir in "$DAOS_INSTALL_DIR" "$DAOS_SOURCE_DIR/build"
 do
 	if [[ -d "$dir" && "$CMD" == install ]]; then
-		[[ -n "$FORCE_INSTALL" ]] || fatal "DAOS already installed: use install --force or update."
-		info "Removing old DAOS install directory $dir"
+		"$FORCE_INSTALL" || fatal "DAOS already installed: use install --force or update."
+		info "Removing old DAOS directory $dir"
 		if ! run /bin/rm -fr "$dir" ; then
 			fatal "Old install directory $dir could not be removed"
 		fi
-
 	fi
 done
 
+check_cmds $SCONS_EXE $MODULE_EXE $CLUSH_EXE $WGET_EXE $MAKE_EXE $CMAKE_EXE
+
 if [[ ! -d "$DAOS_INSTALL_DIR" ]] ; then
 	info "Creating DAOS install directory $DAOS_INSTALL_DIR"
 	if ! run mkdir -p "$DAOS_INSTALL_DIR" ; then
@@ -205,7 +219,7 @@ case $CMD in
 esac
 
 info "Building and installing DAOS from source tree $PWD to $DAOS_INSTALL_DIR"
-if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE=debug PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
+if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
 	fatal "DAOS could not be properly build and/or installed"
 fi
 
@@ -242,14 +256,28 @@ run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST rm -f /etc/ld.so.cache
 run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST ldconfig
 {% endif %}
 
-if [[ $MPICH_PATH ]] ; then
+if [[ ! -f "$DAOS_INSTALL_DIR/bin/activate" ]] ; then
+	info "Installing avocado"
+	run $PYTHON_EXE -m venv "$DAOS_INSTALL_DIR"
+	source "$DAOS_INSTALL_DIR/bin/activate"
+	run $PIP_EXE install --upgrade pip
+	run $PIP_EXE install -r "$DAOS_SOURCE_DIR/requirements.txt"
+	set +u
+	deactivate
+	set -u
+	for file in "$DAOS_SOURCE_DIR"/.build_vars* ; do
+		ln -fs $file "$DAOS_INSTALL_DIR/lib/daos"
+	done
+fi
+
+if [[ ${MPICH_PATH:+x} ]] ; then
 	info "Building and installing MPICH from source tree $MPICH_PATH to $DAOS_INSTALL_DIR"
 
 	if [[ ! -f "$MPICH_PATH/mpich-doxygen.in" ]] ; then
 		fatal "Invalid MPICH directory: path=$MPICH_PATH"
 	fi
 
-	if [[ "$CMD" == install && -n "$FORCE_INSTALL" ]] ; then
+	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
 		info "Removing old MPICH install"
 		if ! run rm -fr "$MPICH_PATH/build" "$MPICH_PATH/configure" ; then
 			fatal "Old install of MPICH could not be removed"
@@ -336,17 +364,17 @@ if [[ $MPICH_PATH ]] ; then
 		mkdir -p /usr/share/modulefiles/mpi
 		ln -fs "$DAOS_INSTALL_DIR/share/modulefiles/mpich-x86_64" /usr/share/modulefiles/mpi/mpich-x86_64
 		EOF
-	} | run $CLUSH_EXE $CLUSH_OPTS -l root -w $CLIENTS_LIST bash -s
+	} | run $CLUSH_EXE $CLUSH_OPTS -l root -w $CLIENTS_LIST -w $SERVERS_LIST bash -s
 fi
 
-if [[ $HDF5_PATH ]] ; then
+if [[ ${HDF5_PATH:+x} ]] ; then
 	info "Building and installing HDF5 from source tree $HDF5_PATH to $DAOS_INSTALL_DIR"
 
 	if [[ ! -f "$HDF5_PATH/config/libhdf5.pc.in" ]] ; then
 		fatal "Invalid HDF5 directory: path=$HDF5_PATH"
 	fi
 
-	if [[ "$CMD" == install && -n "$FORCE_INSTALL" ]] ; then
+	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
 		info "Removing old HDF5 install"
 		if ! run rm -fr "$HDF5_PATH/build" "$HDF5_PATH/configure" ; then
 			fatal "Old install of HDF5 could not be removed"
@@ -382,14 +410,14 @@ if [[ $HDF5_PATH ]] ; then
 	popd > /dev/null
 fi
 
-if [[ $VOL_DAOS_PATH ]] ; then
+if [[ ${VOL_DAOS_PATH:+x} ]] ; then
 	info "Building and installing DAOS HDF5 Volume from source tree $VOL_DAOS_PATH to $DAOS_INSTALL_DIR"
 
 	if [[ ! -f "$VOL_DAOS_PATH/CMake/hdf5_vol_daos.pc.in" ]] ; then
 		fatal "Invalid DAOS HDF5 Volume directory: path=$VOL_DAOS_PATH"
 	fi
 
-	if [[ "$CMD" == install && -n "$FORCE_INSTALL" ]] ; then
+	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
 		info "Removing old DAOS HDF5 Volume install"
 		if ! run rm -fr "$VOL_DAOS_PATH/build" ; then
 			fatal "Old install of HDF5 could not be removed"
@@ -428,14 +456,14 @@ if [[ $VOL_DAOS_PATH ]] ; then
 	popd > /dev/null
 fi
 
-if [[ $MPI_FILE_UTILS_PATH ]] ; then
+if [[ ${MPI_FILE_UTILS_PATH:+x} ]] ; then
 	info "Building and installing MPI File Utils from source tree $MPI_FILE_UTILS_PATH to $DAOS_INSTALL_DIR"
 
 	if [[ ! -f "$MPI_FILE_UTILS_PATH/mpifileutils.spec" ]] ; then
 		fatal "Invalid MPI File Utils directory: path=$MPI_FILE_UTILS_PATH"
 	fi
 
-	if [[ "$CMD" == install && -n "$FORCE_INSTALL" ]] ; then
+	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
 		info "Removing old MPI File Utils install"
 		if ! run rm -fr "$MPI_FILE_UTILS_PATH/deps" "$MPI_FILE_UTILS_PATH/build" "$MPI_FILE_UTILS_PATH/configure" ; then
 			fatal "Old install of MPI File Utils could not be removed"
@@ -499,14 +527,14 @@ if [[ $MPI_FILE_UTILS_PATH ]] ; then
 	popd > /dev/null
 fi
 
-if [[ $IOR_PATH ]] ; then
+if [[ ${IOR_PATH:+x} ]] ; then
 	info "Building and installing IOR from source tree $IOR_PATH to $DAOS_INSTALL_DIR"
 
 	if [[ ! -f "$IOR_PATH/src/ior.c" ]] ; then
 		fatal "Invalid IOR directory: path=$IOR_PATH"
 	fi
 
-	if [[ "$CMD" == install && -n "$FORCE_INSTALL" ]] ; then
+	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
 		info "Removing old IOR install"
 		if ! run rm -fr "$IOR_PATH/build" "$IOR_PATH/configure" ; then
 			fatal "Old install of IOR could not be removed"
@@ -539,6 +567,32 @@ if [[ $IOR_PATH ]] ; then
 	popd > /dev/null
 fi
 
+if [[ ${FIO_PATH:+x} ]] ; then
+	info "Building and installing FIO from source tree $FIO_PATH to $DAOS_INSTALL_DIR"
+
+	if [[ ! -f "$FIO_PATH/fio.c" ]] ; then
+		fatal "Invalid FIO: path=$FIO_PATH"
+	fi
+
+	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
+		info "Removing old FIO install"
+		if ! run rm -fr "$FIO_PATH/build" ; then
+			fatal "Old install of FIO could not be removed"
+		fi
+	fi
+
+	run mkdir -p "$FIO_PATH/build"
+	pushd "$FIO_PATH/build" > /dev/null
+
+	if [[ ! -f "$FIO_PATH/build/Makefile" ]] ; then
+		run env CFLAGS="-I$DAOS_INSTALL_DIR/include" LDFLAGS="-L$DAOS_INSTALL_DIR/lib64" ../configure --prefix="$DAOS_INSTALL_DIR"
+	fi
+
+	run env CFLAGS="-g -O0" make -j $(nproc) install
+
+	popd > /dev/null
+fi
+
 info "Updating dynamic linker configuration"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
 run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST rm -f /etc/ld.so.cache
diff --git a/utils/ansible/ftest/vars/Rocky8.yml b/utils/ansible/ftest/vars/Rocky8.yml
index aecb5478787..3e03527ee4e 100644
--- a/utils/ansible/ftest/vars/Rocky8.yml
+++ b/utils/ansible/ftest/vars/Rocky8.yml
@@ -6,11 +6,14 @@ daos_openmpi_module_path: /usr/share/modulefiles/mpi/openmpi-x86_64
 daos_base_deps:
   rpms:
     - bash
+    - capstone
     - gdb
     - glibc
     - jq
     - lbzip2
-    - mercury
+    - numactl
+    - numactl-libs
+    - numatop
     - platform-python
     - python36
     - systemd
@@ -18,27 +21,45 @@ daos_base_deps:
 
 daos_dev_deps:
   rpms:
+    - argobots-devel
+    - capstone-devel
+    - bzip2-devel
+    - clustershell
     - cmake
     - createrepo
+    - daos-raft-devel
     - dnf-plugins-core
+    - fuse3-devel
+    - gcc-c++
+    - gcc-gfortran
+    - golang
     - libaec-devel
+    - libattr-devel
+    - libcmocka-devel
+    - libevent-devel
+    - libyaml-devel
     - meson
+    - ninja-build
+    - openssl-devel
+    - python3.11
+    - python3.11-devel
     - python3-clustershell
+    - python3-defusedxml
+    - python3-distro
+    - python3-jira
+    - python3-junit_xml
+    - python3-pip
+    - python3-pyelftools
+    - python3-scons
+    - python3-tabulate
+    - python3-wheel
+    - python3-pyxattr
+    - python3-pyyaml
     - rpmdevtools
     - rpm-build
+    - spdk-devel
+    - valgrind-devel
     - wget
-    - libattr-devel
-    - bzip2-devel
-  pips:
-    - "avocado=={{ daos_avocado_version | default(daos_avocado_version_default) }}"
-    - "avocado-framework=={{ daos_avocado_framework_version | default(daos_avocado_framework_version_default) }}"
-    - "avocado-framework-plugin-glib=={{ daos_avocado_framework_version | default(daos_avocado_framework_version_default) }}"
-    - "avocado-framework-plugin-golang=={{ daos_avocado_framework_version | default(daos_avocado_framework_version_default) }}"
-    - "avocado-framework-plugin-loader-yaml=={{ daos_avocado_framework_version | default(daos_avocado_framework_version_default) }}"
-    - "avocado-framework-plugin-result-html=={{ daos_avocado_framework_version | default(daos_avocado_framework_version_default) }}"
-    - "avocado-framework-plugin-result-upload=={{ daos_avocado_framework_version | default(daos_avocado_framework_version_default) }}"
-    - "avocado-framework-plugin-varianter-yaml-to-mux=={{ daos_avocado_framework_version | default(daos_avocado_framework_version_default) }}"
-    - pyelftools
 
 daos_client_deps:
   rpms:
@@ -48,6 +69,7 @@ daos_client_deps:
     - libaec
     - libuuid
     - lz4-libs
+    - numactl-devel
     - systemd
     - util-linux
 
@@ -56,11 +78,13 @@ daos_server_deps:
     - compat-hwloc1
     - dpdk
     - ipmctl
-    - libfabric
     - libipmctl
     - libpmemobj
     - libuuid
     - lz4-libs
     - ndctl
-    - numactl-libs
     - spdk-tools
+
+mlnx_deps:
+  rpms:
+    - mlnx-ofed-all
diff --git a/utils/ansible/ftest/vars/defaults.yml b/utils/ansible/ftest/vars/defaults.yml
index ef679445e25..8104feac144 100644
--- a/utils/ansible/ftest/vars/defaults.yml
+++ b/utils/ansible/ftest/vars/defaults.yml
@@ -1,4 +1,5 @@
-# Install dependencies of a development node
+# Default variable
 
-daos_avocado_version_default: 2.4.3
-daos_avocado_framework_version_default: 82
+daos_ofi_provider_default: ofi+tcp;ofi_rxm
+daos_build_mode_default: DEBUG
+daos_hugepages_nb_default: 4096

From 07b4777dfde638b92be568c282d84a92e4a17882 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 02/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Replace scons-3 with scons
- Add default parameter explanation
- Miscellaneous fixes and improvements

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/inventory-sample.yml      | 28 ++++++++--
 .../ansible/ftest/templates/daos-launch.sh.j2 | 40 +++++++-------
 .../ftest/templates/daos-launch_nlt.sh.j2     | 46 +++++++++-------
 utils/ansible/ftest/templates/daos-make.sh.j2 | 53 +++++++++++--------
 4 files changed, 102 insertions(+), 65 deletions(-)

diff --git a/utils/ansible/ftest/inventory-sample.yml b/utils/ansible/ftest/inventory-sample.yml
index e81d8376268..da7e7b9f304 100644
--- a/utils/ansible/ftest/inventory-sample.yml
+++ b/utils/ansible/ftest/inventory-sample.yml
@@ -1,20 +1,40 @@
 # Sample inventory of a DAOS functional platform running on the wolf cluster
+#
+# Detailled information on the content of this ansible inventory file is available in the README.md
+# file of the directory containing this first file.
 
 all:
   vars:
+    #
+    # Mandatory variable defining the shared directory used to install DAOS.
     daos_runtime_dir: /home/foo/daos
-    # daos_ofi_provider: ofi+tcp;ofi_rxm
+    #
+    # Optional variable defining the network provider to use by the DAOS engines.  This variable is
+    # also used to define which network driver rpms have to be install.
+    #  default: ofi+tcp;ofi_rxm
+    #daos_ofi_provider: ofi+tcp;ofi_rxm
   children:
     daos_dev:
       vars:
+        #
+        # Mandatory variable defining the path of the directory containing the DAOS source code.
         daos_source_dir: /home/foo/work/daos
-        # daos_ofi_interface: eth0
-        # daos_build_mode: DEBUG
+        #
+        # Optional variable defining the network interface to use with the launch.py python script.
+        #  default: None
+        #daos_ofi_interface: eth0
+        #
+        # Optional variable defining in which mode DAOS binaries have to be compiled.
+        #  default: DEBUG
+        #daos_build_mode: DEBUG
       hosts:
         wolf-666:
     daos_servers:
       vars:
-        daos_hugepages_nb: 8182
+        #
+        # Optional variable defining the number of hugepages to be allocated by the linux kernel.
+        #  default: 4096
+        #daos_hugepages_nb: 8182
       hosts:
         wolf-666:
         wolf-999:
diff --git a/utils/ansible/ftest/templates/daos-launch.sh.j2 b/utils/ansible/ftest/templates/daos-launch.sh.j2
index d008beba1e2..223ea08fde5 100644
--- a/utils/ansible/ftest/templates/daos-launch.sh.j2
+++ b/utils/ansible/ftest/templates/daos-launch.sh.j2
@@ -17,7 +17,7 @@ SERVERS_LIST="{{ groups["daos_servers"] | join(',') }}"
 CLIENTS_LIST="{{ groups["daos_clients"] | join(',') }}"
 {% endif %}
 
-PYTHON_EXE=python3
+PYTHON_EXE=python
 CLUSH_EXE=clush
 MODULE_EXE=module
 
@@ -150,13 +150,13 @@ function cleanup
 			rmdir "\$target"
 		done
 		EOF
-	} | $CLUSH_EXE -l root -w $CLIENTS_LIST bash
+	} | run $CLUSH_EXE -l root -w $CLIENTS_LIST bash
 {% endif %}
 
 	info "Clean up of servers $SERVERS_LIST"
-	$CLUSH_EXE -l root -w $SERVERS_LIST sudo systemctl stop daos_server
-	$CLUSH_EXE -l root -w $SERVERS_LIST rm -rf /var/run/daos_server
-	$CLUSH_EXE -l root -w $SERVERS_LIST mkdir -p /var/run/daos_server
+	run $CLUSH_EXE -l root -w $SERVERS_LIST sudo systemctl stop daos_server
+	run $CLUSH_EXE -l root -w $SERVERS_LIST rm -rf /var/run/daos_server
+	run $CLUSH_EXE -l root -w $SERVERS_LIST mkdir -p /var/run/daos_server
 
 	info "Stopping all DAOS processes"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
@@ -164,7 +164,7 @@ function cleanup
 {% else %}
 	CLUSH_NODESET="-w $SERVERS_LIST"
 {% endif %}
-	$CLUSH_EXE -l root $CLUSH_NODESET 'bash -c "killall -q -9 avocado orterun mpirun orted daos_server daos_io_server daos_agent || true"'
+	run $CLUSH_EXE -l root $CLUSH_NODESET 'bash -c "killall -q -9 avocado orterun mpirun orted daos_server daos_io_server daos_agent || true"'
 
 	info "Cleaning mount points of servers"
 	for host in {{ groups["daos_servers"] | join(' ') }}
@@ -172,26 +172,26 @@ function cleanup
 		for index in {0..3}
 		do
 			mountpoint="/mnt/daos$index"
-			if ssh root@$host mountpoint -q "$mountpoint" ; then
+			if run ssh root@$host mountpoint -q "$mountpoint" ; then
 				info "Cleaning mount points $mountpoint of host $host"
-				ssh root@$host rm -fr $mountpoint/*
-				ssh root@$host umount $mountpoint
+				run ssh root@$host rm -fr $mountpoint/*
+				run ssh root@$host umount $mountpoint
 			fi
 		done
 
 		for index in {0..1}
 		do
 			bdev="/dev/pmem$index"
-			if ssh root@$host test -b "$bdev" ; then
+			if run ssh root@$host test -b "$bdev" ; then
 				info "Cleaning persistent memory $bdev of host $host"
-				ssh root@$host wipefs -a "$bdev"
+				run ssh root@$host wipefs -a "$bdev"
 			fi
 		done
 	done
 
 	info "Cleaning huge pages"
-	$CLUSH_EXE -l root -w $SERVERS_LIST ipcrm --all=shm
-	$CLUSH_EXE -l root -w $SERVERS_LIST /bin/rm -f "/dev/hugepages/spdk_*"
+	run $CLUSH_EXE -l root -w $SERVERS_LIST ipcrm --all=shm
+	run $CLUSH_EXE -l root -w $SERVERS_LIST /bin/rm -f "/dev/hugepages/spdk_*"
 }
 
 {% raw %}
@@ -225,7 +225,7 @@ do
 	esac
 done
 
-check_cmds $PYTHON_EXE $CLUSH_EXE $MODULE_EXE
+check_cmds $CLUSH_EXE $MODULE_EXE
 
 if "$START_CLEANUP" ; then
 	cleanup
@@ -235,8 +235,8 @@ info "Setting simulation environment"
 export PYTHONPATH="$DAOS_INSTALL_DIR/lib64/python3.6/site-packages:$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest/util"
 cd "$DAOS_SOURCE_DIR/utils/sl"
 source setup_local.sh
-if $MODULE_EXE is-avail mpi/mpich-x86_64 ; then
-	$MODULE_EXE load mpi/mpich-x86_64
+if run $MODULE_EXE is-avail mpi/mpich-x86_64 ; then
+	run $MODULE_EXE load mpi/mpich-x86_64
 fi
 
 info "Parametrization of the simulation"
@@ -248,12 +248,16 @@ common_opts="-ar --provider={{ daos_ofi_provider | default(daos_ofi_provider_def
 {% endif %}
 
 info "Starting functional test simulation"
+source "$DAOS_INSTALL_DIR/bin/activate"
 {% if daos_ofi_interface is defined %}
-env OFI_INTERFACE=$OFI_INTERFACE $PYTHON_EXE launch.py $common_opts "${args[@]}"
+run env OFI_INTERFACE=$OFI_INTERFACE $PYTHON_EXE launch.py $common_opts "${args[@]}"
 {% else %}
-$PYTHON_EXE launch.py $common_opts "${args[@]}"
+run $PYTHON_EXE launch.py $common_opts "${args[@]}"
 {% endif %}
 rc=$?
+set +u
+deactivate
+set -u
 
 if "$END_CLEANUP" ; then
 	cleanup
diff --git a/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2 b/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
index 548d81009b8..6f2e1fb13ae 100644
--- a/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
+++ b/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
@@ -13,7 +13,7 @@ DAOS_SOURCE_DIR="{{ daos_source_dir }}"
 OFI_INTERFACE="{{ daos_ofi_interface }}"
 {% endif %}
 
-PYTHON_EXE=python3
+PYTHON_EXE=python
 MODULE_EXE=module
 
 ANSI_COLOR_BLACK=30
@@ -122,37 +122,37 @@ function run
 
 function cleanup
 {
-	info "Clean up of servers $SERVERS_LIST"
-	sudo systemctl stop daos_server
-	sudo rm -rf /var/run/daos_server
-	sudo mkdir -p /var/run/daos_server
+	info "Clean up of local server"
+	run sudo systemctl stop daos_server
+	run sudo rm -rf /var/run/daos_server
+	run sudo mkdir -p /var/run/daos_server
 
 	info "Stopping all DAOS related processes"
-	sudo killall -q -9 avocado orterun mpirun orted daos_server daos_io_server daos_agent
+	run sudo killall -q -9 avocado orterun mpirun orted daos_server daos_io_server daos_agent
 
-	info "Cleaning mount points of servers"
+	info "Cleaning mount points of local server"
 	for index in {0..3}
 	do
 		mountpoint="/mnt/daos$index"
-		if mountpoint -q "$mountpoint" ; then
+		if run mountpoint -q "$mountpoint" ; then
 			info "Cleaning mount points $mountpoint of host $host"
-			sudo rm -fr $mountpoint/*
-			sudo umount $mountpoint
+			run sudo rm -fr $mountpoint/*
+			run sudo umount $mountpoint
 		fi
 	done
 
 	for index in {0..1}
 	do
 		bdev="/dev/pmem$index"
-		if test -b "$bdev" ; then
+		if run test -b "$bdev" ; then
 			info "Cleaning persistent memory $bdev of host $host"
-			sudo wipefs -a "$bdev"
+			run sudo wipefs -a "$bdev"
 		fi
 	done
 
 	info "Cleaning huge pages"
-	sudo ipcrm --all=shm
-	sudo /bin/rm -f "/dev/hugepages/spdk_*"
+	run sudo ipcrm --all=shm
+	run sudo /bin/rm -f "/dev/hugepages/spdk_*"
 }
 
 {% raw %}
@@ -186,32 +186,38 @@ do
 	esac
 done
 
-check_cmds $PYTHON_EXE $CLUSH_EXE $MODULE_EXE
+check_cmds $MODULE_EXE
 
 if "$START_CLEANUP" ; then
 	cleanup
 
 	info "Cleaning NLT logs"
-	/bin/rm -f "$DAOS_SOURCE_DIR"/dnt.* "$DAOS_SOURCE_DIR"/nlt-* "$DAOS_SOURCE_DIR"/vgcore.*
+	run /bin/rm -f "$DAOS_SOURCE_DIR"/dnt.* "$DAOS_SOURCE_DIR"/nlt-* "$DAOS_SOURCE_DIR"/vgcore.*
 fi
 
 info "Setting simulation environment"
 export PYTHONPATH="$DAOS_INSTALL_DIR/lib64/python3.6/site-packages:$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest/util"
 cd "$DAOS_SOURCE_DIR/utils/sl"
 source setup_local.sh
-if $MODULE_EXE is-avail mpi/mpich-x86_64 ; then
-	$MODULE_EXE load mpi/mpich-x86_64
+run sudo -E "$DAOS_SOURCE_DIR/utils/setup_daos_server_helper.sh"
+run chmod -x "$DAOS_INSTALL_DIR/bin/daos_server_helper"
+if run $MODULE_EXE is-avail mpi/mpich-x86_64 ; then
+	run $MODULE_EXE load mpi/mpich-x86_64
 fi
 
 info "Starting NLT tests"
+source "$DAOS_INSTALL_DIR/bin/activate"
 pushd "$DAOS_SOURCE_DIR" > /dev/null
 {% if daos_ofi_interface is defined %}
-env OFI_INTERFACE=$OFI_INTERFACE $PYTHON_EXE utils/node_local_test.py $common_opts "${args[@]}"
+run env OFI_INTERFACE=$OFI_INTERFACE $PYTHON_EXE utils/node_local_test.py "${args[@]}"
 {% else %}
-$PYTHON_EXE launch.py $common_opts "${args[@]}"
+run $PYTHON_EXE utils/node_local_test.py "${args[@]}"
 {% endif %}
 rc=$?
 popd > /dev/null
+set +u
+deactivate
+set -u
 
 if "$END_CLEANUP" ; then
 	cleanup
diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index 34ac736b537..cae974ed464 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -16,7 +16,7 @@ CLIENTS_LIST="{{ groups["daos_clients"] | join(',') }}"
 {% endif %}
 BUILD_TYPE="{{ daos_build_mode | default(daos_build_mode_default) }}"
 
-SCONS_EXE=scons-3
+SCONS_EXE=scons
 SSH_EXE=ssh
 CLUSH_EXE=clush
 NODESET_EXE=nodeset
@@ -176,7 +176,7 @@ done
 CMD="$1"
 ${VALID_CMDS[$CMD]} || fatal "Invalid command \"$CMD\": use install or update"
 
-check_cmds $SSH_EXE $CLUSH_EXE $NODESET_EXE
+check_cmds $SSH_EXE $WGET_EXE $CLUSH_EXE $NODESET_EXE $MODULE_EXE $MAKE_EXE $CMAKE_EXE
 
 set -e
 
@@ -201,8 +201,6 @@ do
 	fi
 done
 
-check_cmds $SCONS_EXE $MODULE_EXE $CLUSH_EXE $WGET_EXE $MAKE_EXE $CMAKE_EXE
-
 if [[ ! -d "$DAOS_INSTALL_DIR" ]] ; then
 	info "Creating DAOS install directory $DAOS_INSTALL_DIR"
 	if ! run mkdir -p "$DAOS_INSTALL_DIR" ; then
@@ -210,6 +208,20 @@ if [[ ! -d "$DAOS_INSTALL_DIR" ]] ; then
 	fi
 fi
 
+if [[ ! -f "$DAOS_INSTALL_DIR/bin/activate" ]] ; then
+	info "Installing avocado"
+	run $PYTHON_EXE -m venv "$DAOS_INSTALL_DIR"
+	source "$DAOS_INSTALL_DIR/bin/activate"
+	run $PIP_EXE install --upgrade pip
+	run $PIP_EXE install -r "$DAOS_SOURCE_DIR/requirements.txt"
+	set +u
+	deactivate
+	set -u
+	info "Customize venv for DAOS install"
+	run rm -fv "$DAOS_INSTALL_DIR/lib64"
+	run mkdir -pv "$DAOS_INSTALL_DIR/lib64"
+fi
+
 cd "$DAOS_SOURCE_DIR"
 SCONS_OPTS="-j $JOBS_NB --implicit-deps-changed"
 case $CMD in
@@ -219,12 +231,21 @@ case $CMD in
 esac
 
 info "Building and installing DAOS from source tree $PWD to $DAOS_INSTALL_DIR"
+source "$DAOS_INSTALL_DIR/bin/activate"
 if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
 	fatal "DAOS could not be properly build and/or installed"
 fi
+set +u
+deactivate
+set -u
 
 info "Finalizing install of ftest setup scripts"
-run ln -vfs "$DAOS_SOURCE_DIR/.build_vars.sh" "$DAOS_SOURCE_DIR/utils/sl/.build_vars-Linux.sh"
+for file_path in "$DAOS_SOURCE_DIR"/.build_vars*
+do
+	file_name=$(basename $file_path)
+	rm -fv "$DAOS_INSTALL_DIR/lib/daos/$file_name"
+	run cp -av "$file_path" "$DAOS_INSTALL_DIR/lib/daos/$file_name"
+done
 
 info "Finalizing spdk install"
 for item in $(find "$DAOS_INSTALL_DIR/share/daos" -type f)
@@ -256,20 +277,6 @@ run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST rm -f /etc/ld.so.cache
 run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST ldconfig
 {% endif %}
 
-if [[ ! -f "$DAOS_INSTALL_DIR/bin/activate" ]] ; then
-	info "Installing avocado"
-	run $PYTHON_EXE -m venv "$DAOS_INSTALL_DIR"
-	source "$DAOS_INSTALL_DIR/bin/activate"
-	run $PIP_EXE install --upgrade pip
-	run $PIP_EXE install -r "$DAOS_SOURCE_DIR/requirements.txt"
-	set +u
-	deactivate
-	set -u
-	for file in "$DAOS_SOURCE_DIR"/.build_vars* ; do
-		ln -fs $file "$DAOS_INSTALL_DIR/lib/daos"
-	done
-fi
-
 if [[ ${MPICH_PATH:+x} ]] ; then
 	info "Building and installing MPICH from source tree $MPICH_PATH to $DAOS_INSTALL_DIR"
 
@@ -381,7 +388,7 @@ if [[ ${HDF5_PATH:+x} ]] ; then
 		fi
 	fi
 
-	$MODULE_EXE load mpi/mpich-x86_64
+	run $MODULE_EXE load mpi/mpich-x86_64
 
 	if [[ ! -f "$HDF5_PATH/configure" ]] ; then
 		pushd "$HDF5_PATH" > /dev/null
@@ -424,7 +431,7 @@ if [[ ${VOL_DAOS_PATH:+x} ]] ; then
 		fi
 	fi
 
-	$MODULE_EXE load mpi/mpich-x86_64
+	run $MODULE_EXE load mpi/mpich-x86_64
 
 	run mkdir -p "$VOL_DAOS_PATH/build"
 	pushd "$VOL_DAOS_PATH/build" > /dev/null
@@ -470,7 +477,7 @@ if [[ ${MPI_FILE_UTILS_PATH:+x} ]] ; then
 		fi
 	fi
 
-	$MODULE_EXE load mpi/mpich-x86_64
+	run $MODULE_EXE load mpi/mpich-x86_64
 
 	run mkdir -p "$MPI_FILE_UTILS_PATH/deps" "$MPI_FILE_UTILS_PATH/build"
 
@@ -541,7 +548,7 @@ if [[ ${IOR_PATH:+x} ]] ; then
 		fi
 	fi
 
-	$MODULE_EXE load mpi/mpich-x86_64
+	run $MODULE_EXE load mpi/mpich-x86_64
 
 	if [[ ! -f "$IOR_PATH/configure" ]] ; then
 		pushd "$IOR_PATH" > /dev/null

From fa2109a39924a5d4d4fdb71caddef9db0a0638c2 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 03/17] DAOS-15603: Support of FIO with ansible ftest script

Fix code spell

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/inventory-sample.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/ansible/ftest/inventory-sample.yml b/utils/ansible/ftest/inventory-sample.yml
index da7e7b9f304..7aa140b6f3c 100644
--- a/utils/ansible/ftest/inventory-sample.yml
+++ b/utils/ansible/ftest/inventory-sample.yml
@@ -1,6 +1,6 @@
 # Sample inventory of a DAOS functional platform running on the wolf cluster
 #
-# Detailled information on the content of this ansible inventory file is available in the README.md
+# Detailed information on the content of this ansible inventory file is available in the README.md
 # file of the directory containing this first file.
 
 all:

From d9fa641e269400d83040623865da83f7e2122175 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 04/17] DAOS-15603: Support of FIO with ansible ftest script

Fix yaml lint check

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/inventory-sample.yml | 76 ++++++++++++------------
 1 file changed, 38 insertions(+), 38 deletions(-)

diff --git a/utils/ansible/ftest/inventory-sample.yml b/utils/ansible/ftest/inventory-sample.yml
index 7aa140b6f3c..5570d244575 100644
--- a/utils/ansible/ftest/inventory-sample.yml
+++ b/utils/ansible/ftest/inventory-sample.yml
@@ -3,41 +3,41 @@
 # Detailed information on the content of this ansible inventory file is available in the README.md
 # file of the directory containing this first file.
 
-all:
-  vars:
-    #
-    # Mandatory variable defining the shared directory used to install DAOS.
-    daos_runtime_dir: /home/foo/daos
-    #
-    # Optional variable defining the network provider to use by the DAOS engines.  This variable is
-    # also used to define which network driver rpms have to be install.
-    #  default: ofi+tcp;ofi_rxm
-    #daos_ofi_provider: ofi+tcp;ofi_rxm
-  children:
-    daos_dev:
-      vars:
-        #
-        # Mandatory variable defining the path of the directory containing the DAOS source code.
-        daos_source_dir: /home/foo/work/daos
-        #
-        # Optional variable defining the network interface to use with the launch.py python script.
-        #  default: None
-        #daos_ofi_interface: eth0
-        #
-        # Optional variable defining in which mode DAOS binaries have to be compiled.
-        #  default: DEBUG
-        #daos_build_mode: DEBUG
-      hosts:
-        wolf-666:
-    daos_servers:
-      vars:
-        #
-        # Optional variable defining the number of hugepages to be allocated by the linux kernel.
-        #  default: 4096
-        #daos_hugepages_nb: 8182
-      hosts:
-        wolf-666:
-        wolf-999:
-    daos_clients:
-      hosts:
-        wolf-999:
+# all:
+#   vars:
+#     #
+#     # Mandatory variable defining the shared directory used to install DAOS.
+#     daos_runtime_dir: /home/foo/daos
+#     #
+#     # Optional variable defining the network provider to use by the DAOS engines.  This variable is
+#     # also used to define which network driver rpms have to be install.
+#     #  default: ofi+tcp;ofi_rxm
+#     daos_ofi_provider: ofi+tcp;ofi_rxm
+#   children:
+#     daos_dev:
+#       vars:
+#         #
+#         # Mandatory variable defining the path of the directory containing the DAOS source code.
+#         daos_source_dir: /home/foo/work/daos
+#         #
+#         # Optional variable defining the network interface to use with the launch.py python script.
+#         #  default: None
+#         daos_ofi_interface: eth0
+#         #
+#         # Optional variable defining in which mode DAOS binaries have to be compiled.
+#         #  default: DEBUG
+#         daos_build_mode: DEBUG
+#       hosts:
+#         wolf-666:
+#     daos_servers:
+#       vars:
+#         #
+#         # Optional variable defining the number of hugepages to be allocated by the linux kernel.
+#         #  default: 4096
+#         daos_hugepages_nb: 8182
+#       hosts:
+#         wolf-666:
+#         wolf-999:
+#     daos_clients:
+#       hosts:
+#         wolf-999:

From 712254ed8a27408015eb10c4949a6a5831c9b39d Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 05/17] DAOS-15603: Support of FIO with ansible ftest script

Fix yaml lint check

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/inventory-sample.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/utils/ansible/ftest/inventory-sample.yml b/utils/ansible/ftest/inventory-sample.yml
index 5570d244575..e7cd6b0e7dc 100644
--- a/utils/ansible/ftest/inventory-sample.yml
+++ b/utils/ansible/ftest/inventory-sample.yml
@@ -9,8 +9,8 @@
 #     # Mandatory variable defining the shared directory used to install DAOS.
 #     daos_runtime_dir: /home/foo/daos
 #     #
-#     # Optional variable defining the network provider to use by the DAOS engines.  This variable is
-#     # also used to define which network driver rpms have to be install.
+#     # Optional variable defining the network provider to use by the DAOS engines.  This variable
+#     # is also used to define which network driver rpms have to be install.
 #     #  default: ofi+tcp;ofi_rxm
 #     daos_ofi_provider: ofi+tcp;ofi_rxm
 #   children:
@@ -20,7 +20,8 @@
 #         # Mandatory variable defining the path of the directory containing the DAOS source code.
 #         daos_source_dir: /home/foo/work/daos
 #         #
-#         # Optional variable defining the network interface to use with the launch.py python script.
+#         # Optional variable defining the network interface to use with the launch.py python
+#         # script.
 #         #  default: None
 #         daos_ofi_interface: eth0
 #         #

From ad2c133d7fe2927e9c2ef022e81f60041e1c379d Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 06/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Remove useless set +u before activate

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/templates/daos-launch.sh.j2     | 2 --
 utils/ansible/ftest/templates/daos-launch_nlt.sh.j2 | 2 --
 utils/ansible/ftest/templates/daos-make.sh.j2       | 4 ----
 3 files changed, 8 deletions(-)

diff --git a/utils/ansible/ftest/templates/daos-launch.sh.j2 b/utils/ansible/ftest/templates/daos-launch.sh.j2
index 223ea08fde5..2910f2c57ac 100644
--- a/utils/ansible/ftest/templates/daos-launch.sh.j2
+++ b/utils/ansible/ftest/templates/daos-launch.sh.j2
@@ -255,9 +255,7 @@ run env OFI_INTERFACE=$OFI_INTERFACE $PYTHON_EXE launch.py $common_opts "${args[
 run $PYTHON_EXE launch.py $common_opts "${args[@]}"
 {% endif %}
 rc=$?
-set +u
 deactivate
-set -u
 
 if "$END_CLEANUP" ; then
 	cleanup
diff --git a/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2 b/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
index 6f2e1fb13ae..bcd88571390 100644
--- a/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
+++ b/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
@@ -215,9 +215,7 @@ run $PYTHON_EXE utils/node_local_test.py "${args[@]}"
 {% endif %}
 rc=$?
 popd > /dev/null
-set +u
 deactivate
-set -u
 
 if "$END_CLEANUP" ; then
 	cleanup
diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index cae974ed464..c4738cdc004 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -214,9 +214,7 @@ if [[ ! -f "$DAOS_INSTALL_DIR/bin/activate" ]] ; then
 	source "$DAOS_INSTALL_DIR/bin/activate"
 	run $PIP_EXE install --upgrade pip
 	run $PIP_EXE install -r "$DAOS_SOURCE_DIR/requirements.txt"
-	set +u
 	deactivate
-	set -u
 	info "Customize venv for DAOS install"
 	run rm -fv "$DAOS_INSTALL_DIR/lib64"
 	run mkdir -pv "$DAOS_INSTALL_DIR/lib64"
@@ -235,9 +233,7 @@ source "$DAOS_INSTALL_DIR/bin/activate"
 if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
 	fatal "DAOS could not be properly build and/or installed"
 fi
-set +u
 deactivate
-set -u
 
 info "Finalizing install of ftest setup scripts"
 for file_path in "$DAOS_SOURCE_DIR"/.build_vars*

From 351fdb02f42c01cd83d4fec92430df307491d590 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 07/17] DAOS-15603: Support of FIO with ansible ftest script

Makes DAOS type of build configurable.

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/README.md                 | 3 ---
 utils/ansible/ftest/inventory-sample.yml      | 4 ----
 utils/ansible/ftest/templates/daos-make.sh.j2 | 7 ++++---
 utils/ansible/ftest/vars/defaults.yml         | 1 -
 4 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/utils/ansible/ftest/README.md b/utils/ansible/ftest/README.md
index 5bc4ab16f04..b83f9566aef 100644
--- a/utils/ansible/ftest/README.md
+++ b/utils/ansible/ftest/README.md
@@ -76,8 +76,6 @@ The inventory should also contain a set of mandatory and optional variables.
 - **daos\_hugepages\_nb**: optional variable (default value: 4096) only used by the nodes of the
   *daos\_servers* group.  This variable defines the number of hugepages to be allocated by the linux
   kernel.
-- **daos\_build\_mode**: optional variable (default value: DEBUG) only used by the nodes of the
-  *daos\_dev* group.  This variable defines in which mode DAOS shall be compiled.
 
 Different file format (e.g. YAML, INI, etc.) and file tree structure are supported to define an
 ansible inventory.  The following simple ansible inventory describe for example in one YAML file
@@ -92,7 +90,6 @@ all:
       vars:
         daos_source_dir: /home/foo/work/daos
         daos_ofi_interface: eth0
-        daos_build_mode: DEBUG
       hosts:
         wolf-666:
     daos_servers:
diff --git a/utils/ansible/ftest/inventory-sample.yml b/utils/ansible/ftest/inventory-sample.yml
index e7cd6b0e7dc..2a6307d3561 100644
--- a/utils/ansible/ftest/inventory-sample.yml
+++ b/utils/ansible/ftest/inventory-sample.yml
@@ -24,10 +24,6 @@
 #         # script.
 #         #  default: None
 #         daos_ofi_interface: eth0
-#         #
-#         # Optional variable defining in which mode DAOS binaries have to be compiled.
-#         #  default: DEBUG
-#         daos_build_mode: DEBUG
 #       hosts:
 #         wolf-666:
 #     daos_servers:
diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index c4738cdc004..b003ed201f7 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -14,7 +14,7 @@ SERVERS_LIST="{{ groups["daos_servers"] | join(',') }}"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
 CLIENTS_LIST="{{ groups["daos_clients"] | join(',') }}"
 {% endif %}
-BUILD_TYPE="{{ daos_build_mode | default(daos_build_mode_default) }}"
+DAOS_BUILD_TYPE=debug
 
 SCONS_EXE=scons
 SSH_EXE=ssh
@@ -149,7 +149,7 @@ function run
 	fi
 }
 
-OPTIONS=$(getopt -o "j:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,fio:,jobs:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
+build-mode=$(getopt -o "j:b:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,fio:,jobs:,build-type:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
 eval set -- "$OPTIONS"
 while true
 do
@@ -160,6 +160,7 @@ do
 		--mpifileutils) MPI_FILE_UTILS_PATH="$2" ; shift 2 ;;
 		--ior) IOR_PATH="$2" ; shift 2 ;;
 		--fio) FIO_PATH="$2" ; shift 2 ;;
+		-b|--build-type) DAOS_BUILD_TYPE="$2" ; shift 2 ;;
 		-j|--jobs) JOBS_NB="$2" ; shift 2 ;;
 		-f|--force) FORCE_INSTALL="true" ; shift 1;;
 		-h|--help) usage ; exit 0;;
@@ -230,7 +231,7 @@ esac
 
 info "Building and installing DAOS from source tree $PWD to $DAOS_INSTALL_DIR"
 source "$DAOS_INSTALL_DIR/bin/activate"
-if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
+if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
 	fatal "DAOS could not be properly build and/or installed"
 fi
 deactivate
diff --git a/utils/ansible/ftest/vars/defaults.yml b/utils/ansible/ftest/vars/defaults.yml
index 8104feac144..bfde27902e4 100644
--- a/utils/ansible/ftest/vars/defaults.yml
+++ b/utils/ansible/ftest/vars/defaults.yml
@@ -1,5 +1,4 @@
 # Default variable
 
 daos_ofi_provider_default: ofi+tcp;ofi_rxm
-daos_build_mode_default: DEBUG
 daos_hugepages_nb_default: 4096

From 92541dd8510778b1b3a94fd1277f90e31c1edef2 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 08/17] DAOS-15603: Support of FIO with ansible ftest script

Fix invalid list expansion

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/tasks/daos_servers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/ansible/ftest/tasks/daos_servers.yml b/utils/ansible/ftest/tasks/daos_servers.yml
index 3ea6973e13e..cb4dd022fb0 100644
--- a/utils/ansible/ftest/tasks/daos_servers.yml
+++ b/utils/ansible/ftest/tasks/daos_servers.yml
@@ -84,4 +84,4 @@
     owner: root
     group: root
     mode: 0755
-  loop: "[''] + {{ range(0, 4, 1) | list }}"
+  loop: "{{ range(0, 4, 1) | list + [''] }}"

From 79615a794810dfe810a571416598c8cf8f9584b4 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 09/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Split virtualenv and DAOS local install

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 .../ansible/ftest/templates/daos-launch.sh.j2 | 13 ++++++++---
 .../ftest/templates/daos-launch_nlt.sh.j2     | 19 +++++++++++----
 utils/ansible/ftest/templates/daos-make.sh.j2 | 23 +++++++++++--------
 3 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/utils/ansible/ftest/templates/daos-launch.sh.j2 b/utils/ansible/ftest/templates/daos-launch.sh.j2
index 2910f2c57ac..632d6c2ebbc 100644
--- a/utils/ansible/ftest/templates/daos-launch.sh.j2
+++ b/utils/ansible/ftest/templates/daos-launch.sh.j2
@@ -8,6 +8,7 @@ CWD="$(realpath "${0%}")"
 CWD="${CWD%/*}"
 
 DAOS_INSTALL_DIR="{{ daos_runtime_dir }}/install"
+DAOS_VENV_DIR="{{ daos_runtime_dir }}/virtualenvs"
 DAOS_SOURCE_DIR="{{ daos_source_dir }}"
 {% if daos_ofi_interface is defined %}
 OFI_INTERFACE="{{ daos_ofi_interface }}"
@@ -231,10 +232,17 @@ if "$START_CLEANUP" ; then
 	cleanup
 fi
 
-info "Setting simulation environment"
-export PYTHONPATH="$DAOS_INSTALL_DIR/lib64/python3.6/site-packages:$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest/util"
+info "Setting python simulation environment"
+source "$DAOS_VENV_DIR/bin/activate"
+if [[ ${PYTHONPATH:+x} ]] ; then
+	export PYTHONPATH="$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest/util:$PYTHONPATH"
+else
+	export PYTHONPATH="$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest/util"
+fi
 cd "$DAOS_SOURCE_DIR/utils/sl"
 source setup_local.sh
+
+info "Setting MPI simulation environment"
 if run $MODULE_EXE is-avail mpi/mpich-x86_64 ; then
 	run $MODULE_EXE load mpi/mpich-x86_64
 fi
@@ -248,7 +256,6 @@ common_opts="-ar --provider={{ daos_ofi_provider | default(daos_ofi_provider_def
 {% endif %}
 
 info "Starting functional test simulation"
-source "$DAOS_INSTALL_DIR/bin/activate"
 {% if daos_ofi_interface is defined %}
 run env OFI_INTERFACE=$OFI_INTERFACE $PYTHON_EXE launch.py $common_opts "${args[@]}"
 {% else %}
diff --git a/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2 b/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
index bcd88571390..82687901ed2 100644
--- a/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
+++ b/utils/ansible/ftest/templates/daos-launch_nlt.sh.j2
@@ -8,6 +8,7 @@ CWD="$(realpath "${0%}")"
 CWD="${CWD%/*}"
 
 DAOS_INSTALL_DIR="{{ daos_runtime_dir }}/install"
+DAOS_VENV_DIR="{{ daos_runtime_dir }}/virtualenvs"
 DAOS_SOURCE_DIR="{{ daos_source_dir }}"
 {% if daos_ofi_interface is defined %}
 OFI_INTERFACE="{{ daos_ofi_interface }}"
@@ -195,18 +196,26 @@ if "$START_CLEANUP" ; then
 	run /bin/rm -f "$DAOS_SOURCE_DIR"/dnt.* "$DAOS_SOURCE_DIR"/nlt-* "$DAOS_SOURCE_DIR"/vgcore.*
 fi
 
-info "Setting simulation environment"
-export PYTHONPATH="$DAOS_INSTALL_DIR/lib64/python3.6/site-packages:$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest/util"
+info "Setting python simulation environment"
+source "$DAOS_VENV_DIR/bin/activate"
+if [[ ${PYTHONPATH:+x} ]] ; then
+	export PYTHONPATH="$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest/util:$PYTHONPATH"
+else
+	export PYTHONPATH="$DAOS_INSTALL_DIR/lib/daos/TESTING/ftest/util"
+fi
 cd "$DAOS_SOURCE_DIR/utils/sl"
 source setup_local.sh
-run sudo -E "$DAOS_SOURCE_DIR/utils/setup_daos_server_helper.sh"
-run chmod -x "$DAOS_INSTALL_DIR/bin/daos_server_helper"
+
+info "Setting MPI simulation environment"
 if run $MODULE_EXE is-avail mpi/mpich-x86_64 ; then
 	run $MODULE_EXE load mpi/mpich-x86_64
 fi
 
+info "Fixing daos_server_helper"
+run sudo -E "$DAOS_SOURCE_DIR/utils/setup_daos_server_helper.sh"
+run chmod -x "$DAOS_INSTALL_DIR/bin/daos_server_helper"
+
 info "Starting NLT tests"
-source "$DAOS_INSTALL_DIR/bin/activate"
 pushd "$DAOS_SOURCE_DIR" > /dev/null
 {% if daos_ofi_interface is defined %}
 run env OFI_INTERFACE=$OFI_INTERFACE $PYTHON_EXE utils/node_local_test.py "${args[@]}"
diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index b003ed201f7..5bda8b93049 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -9,6 +9,7 @@ CWD="${CWD%/*}"
 declare -A VALID_CMDS=([install]=true [update]=true)
 
 DAOS_INSTALL_DIR="{{ daos_runtime_dir }}/install"
+DAOS_VENV_DIR="{{ daos_runtime_dir }}/virtualenvs"
 DAOS_SOURCE_DIR="{{ daos_source_dir }}"
 SERVERS_LIST="{{ groups["daos_servers"] | join(',') }}"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
@@ -149,7 +150,7 @@ function run
 	fi
 }
 
-build-mode=$(getopt -o "j:b:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,fio:,jobs:,build-type:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
+OPTIONS=$(getopt -o "j:b:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,fio:,jobs:,build-type:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
 eval set -- "$OPTIONS"
 while true
 do
@@ -191,7 +192,7 @@ do
 	run $SSH_EXE root@$host mountpoint --quiet "$HOME" || fatal "Home directory '$home' not mounted on $host"
 done
 
-for dir in "$DAOS_INSTALL_DIR" "$DAOS_SOURCE_DIR/build"
+for dir in "$DAOS_INSTALL_DIR" "$DAOS_VENV_DIR" "$DAOS_SOURCE_DIR/build"
 do
 	if [[ -d "$dir" && "$CMD" == install ]]; then
 		"$FORCE_INSTALL" || fatal "DAOS already installed: use install --force or update."
@@ -209,16 +210,20 @@ if [[ ! -d "$DAOS_INSTALL_DIR" ]] ; then
 	fi
 fi
 
-if [[ ! -f "$DAOS_INSTALL_DIR/bin/activate" ]] ; then
+if [[ ! -d "$DAOS_VENV_DIR" ]] ; then
+	info "Creating DAOS python virtualenv directory $DAOS_VENV_DIR"
+	if ! run mkdir -p "$DAOS_VENV_DIR" ; then
+		fatal "DAOS python virtualenv directory $DAOS_VENV_DIR could not be created"
+	fi
+fi
+
+if [[ ! -f "$DAOS_VENV_DIR/bin/activate" ]] ; then
 	info "Installing avocado"
-	run $PYTHON_EXE -m venv "$DAOS_INSTALL_DIR"
-	source "$DAOS_INSTALL_DIR/bin/activate"
+	run $PYTHON_EXE -m venv "$DAOS_VENV_DIR"
+	source "$DAOS_VENV_DIR/bin/activate"
 	run $PIP_EXE install --upgrade pip
 	run $PIP_EXE install -r "$DAOS_SOURCE_DIR/requirements.txt"
 	deactivate
-	info "Customize venv for DAOS install"
-	run rm -fv "$DAOS_INSTALL_DIR/lib64"
-	run mkdir -pv "$DAOS_INSTALL_DIR/lib64"
 fi
 
 cd "$DAOS_SOURCE_DIR"
@@ -230,7 +235,7 @@ case $CMD in
 esac
 
 info "Building and installing DAOS from source tree $PWD to $DAOS_INSTALL_DIR"
-source "$DAOS_INSTALL_DIR/bin/activate"
+source "$DAOS_VENV_DIR/bin/activate"
 if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
 	fatal "DAOS could not be properly build and/or installed"
 fi

From bce8c6cb288ad0605a53f42649b035f6e81d8546 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 10/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Use scons with -C option

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/templates/daos-make.sh.j2 | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index 5bda8b93049..e5f44290c94 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -226,8 +226,7 @@ if [[ ! -f "$DAOS_VENV_DIR/bin/activate" ]] ; then
 	deactivate
 fi
 
-cd "$DAOS_SOURCE_DIR"
-SCONS_OPTS="-j $JOBS_NB --implicit-deps-changed"
+SCONS_OPTS="--directory="$DAOS_SOURCE_DIR" --jobs=$JOBS_NB --implicit-deps-changed"
 case $CMD in
 	install) SCONS_OPTS="$SCONS_OPTS --build-deps=yes --config=force" ;;
 	update) SCONS_OPTS="$SCONS_OPTS --build-deps=no" ;;

From f8c49965c035fd3a39fe24c51e09a44e628bf0d4 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 11/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Remove useless scons option --config-force and --implicit-deps-changed.

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/templates/daos-make.sh.j2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index e5f44290c94..4f448f0459e 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -226,9 +226,9 @@ if [[ ! -f "$DAOS_VENV_DIR/bin/activate" ]] ; then
 	deactivate
 fi
 
-SCONS_OPTS="--directory="$DAOS_SOURCE_DIR" --jobs=$JOBS_NB --implicit-deps-changed"
+SCONS_OPTS="--directory="$DAOS_SOURCE_DIR" --jobs=$JOBS_NB"
 case $CMD in
-	install) SCONS_OPTS="$SCONS_OPTS --build-deps=yes --config=force" ;;
+	install) SCONS_OPTS="$SCONS_OPTS --build-deps=yes" ;;
 	update) SCONS_OPTS="$SCONS_OPTS --build-deps=no" ;;
 	*) fatal "Unexpected case" ;;
 esac

From 8a62b1f000ca6c212e790b7aaddf992c98651969 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 12/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Properly call scons

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/README.md                 | 23 +-----
 utils/ansible/ftest/templates/daos-make.sh.j2 | 70 ++++++++++---------
 2 files changed, 37 insertions(+), 56 deletions(-)

diff --git a/utils/ansible/ftest/README.md b/utils/ansible/ftest/README.md
index b83f9566aef..f326be4c05f 100644
--- a/utils/ansible/ftest/README.md
+++ b/utils/ansible/ftest/README.md
@@ -155,31 +155,11 @@ defined in the inventory. Usage of these two scripts will be detailed in the fol
 ## Installing DAOS Binaries
 
 Building and installing binaries could be done thanks to the generated bash script `daos-make.sh`.
-This last script supports several options and the two sub-commands `install` and `update`.
+This last script supports several options alllowing such as building dependencies or not.
 
 More details on the supported options could be found with running the command with the `--help`
 option.
 
-The `install` sub-command should be used to install/reinstall from scratch all the DAOS binaries and
-their dependencies such as the *spdk* or the *mercury* libraries. For example, the *foo* user could
-use the following command line to install or reinstall the DAOS binaries and its dependencides into
-the `/home/foo/daos/install` directory.
-
-```bash
-/home/foo/daos/daos-make.sh -v -j 32 -f install
-```
-
-When the previous *install* step has been done and the DAOS source code tests have been updated, the
-DAOS binaries could be build and reinstalled thanks to the `update` sub-command.  This last one
-should be far quicker than a full reinstall.
-
-```bash
-/home/foo/daos/daos-make.sh -v -j 32 update
-```
-
-> :bulb: The option `-j 32` (i.e. one per core) seems to be a good compromise between compilation
-> reliability and speed.
-
 ## Launching Functional Tests
 
 Launching the DAOS functional tests could be done thanks to the generated bash script
@@ -192,7 +172,6 @@ with running the command with the `--help` option.
 After successfully installing the DAOS binaries and dependencies, the user *foo* could run all the
 test(s) with the tag `hello_world` thanks to the following command line.
 
-
 ```bash
 /home/foo/daos/daos-launch.sh -v -- --nvme=auto  hello_world
 ```
diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index 4f448f0459e..5d63972d128 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -6,7 +6,6 @@ set -u -o pipefail
 VERSION=0.4
 CWD="$(realpath "${0%}")"
 CWD="${CWD%/*}"
-declare -A VALID_CMDS=([install]=true [update]=true)
 
 DAOS_INSTALL_DIR="{{ daos_runtime_dir }}/install"
 DAOS_VENV_DIR="{{ daos_runtime_dir }}/virtualenvs"
@@ -15,7 +14,6 @@ SERVERS_LIST="{{ groups["daos_servers"] | join(',') }}"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
 CLIENTS_LIST="{{ groups["daos_clients"] | join(',') }}"
 {% endif %}
-DAOS_BUILD_TYPE=debug
 
 SCONS_EXE=scons
 SSH_EXE=ssh
@@ -58,8 +56,10 @@ TRACE_LEVEL_VERBOSE=1
 TRACE_LEVEL_DEBUG=2
 TRACE_LEVEL=$TRACE_LEVEL_STANDARD
 
+DAOS_BUILD_DEPS=false
+DAOS_BUILD_TYPE=debug
 JOBS_NB=1
-FORCE_INSTALL="false"
+FORCE_INSTALL=false
 
 function debug
 {
@@ -115,7 +115,7 @@ function check_cmds
 function usage
 {
 	cat <<- EOF
-		usage: daos-make.sh [OPTIONS] CMD
+		usage: daos-make.sh [OPTIONS]
 
 		Build and install DAOS for running avocado functional tests
 
@@ -126,17 +126,15 @@ function usage
 		       --mpifileutils <path>    Path of mpi file utils source tree
 		       --ior <path>             Path of ior file utils source tree
 		       --fio <path>             Path of fio file utils source tree
+		   -b, --build-type <type>      DAOS build type: debug, release or dev
+		   -d, --deps                   Build DAOS dependencies
 		   -j, --jobs <job>             Allow N jobs at once; one job with no arg
-		   -f, --force                  Overwrite the current install of DAOS
+		   -f, --force                  Overwrite the current install of DAOS and deps
 		   -h, --help                   Show this help message and exit
 		   -V, --version                Show version number
 		   -q, --quiet                  Quiet mode
 		   -v, --verbose                Verbose mode
 		   -D, --debug                  Debug mode
-
-		Commands:
-		   install                      Build and install DAOS
-		   update                       Update current install of DAOS
 	EOF
 }
 
@@ -150,7 +148,7 @@ function run
 	fi
 }
 
-OPTIONS=$(getopt -o "j:b:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,fio:,jobs:,build-type:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
+OPTIONS=$(getopt -o "b:dj:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,fio:,build-type:,deps,jobs:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
 eval set -- "$OPTIONS"
 while true
 do
@@ -162,8 +160,9 @@ do
 		--ior) IOR_PATH="$2" ; shift 2 ;;
 		--fio) FIO_PATH="$2" ; shift 2 ;;
 		-b|--build-type) DAOS_BUILD_TYPE="$2" ; shift 2 ;;
+		-d|--deps) DAOS_BUILD_DEPS=true ; shift 1 ;;
 		-j|--jobs) JOBS_NB="$2" ; shift 2 ;;
-		-f|--force) FORCE_INSTALL="true" ; shift 1;;
+		-f|--force) FORCE_INSTALL=true ; shift 1;;
 		-h|--help) usage ; exit 0;;
 		-V|--version) echo "daos-make.sh version=$VERSION" ; exit 0 ;;
 		-v|--verbose) TRACE_LEVEL=$TRACE_LEVEL_VERBOSE ; shift 1 ;;
@@ -174,10 +173,6 @@ do
 	esac
 done
 
-[[ ${1:+x} ]] || fatal "Command not defined: use install or update"
-CMD="$1"
-${VALID_CMDS[$CMD]} || fatal "Invalid command \"$CMD\": use install or update"
-
 check_cmds $SSH_EXE $WGET_EXE $CLUSH_EXE $NODESET_EXE $MODULE_EXE $MAKE_EXE $CMAKE_EXE
 
 set -e
@@ -192,16 +187,15 @@ do
 	run $SSH_EXE root@$host mountpoint --quiet "$HOME" || fatal "Home directory '$home' not mounted on $host"
 done
 
-for dir in "$DAOS_INSTALL_DIR" "$DAOS_VENV_DIR" "$DAOS_SOURCE_DIR/build"
-do
-	if [[ -d "$dir" && "$CMD" == install ]]; then
-		"$FORCE_INSTALL" || fatal "DAOS already installed: use install --force or update."
+if "$FORCE_INSTALL" ; then
+	for dir in "$DAOS_INSTALL_DIR" "$DAOS_VENV_DIR" "$DAOS_SOURCE_DIR/build"
+	do
 		info "Removing old DAOS directory $dir"
 		if ! run /bin/rm -fr "$dir" ; then
 			fatal "Old install directory $dir could not be removed"
 		fi
-	fi
-done
+	done
+fi
 
 if [[ ! -d "$DAOS_INSTALL_DIR" ]] ; then
 	info "Creating DAOS install directory $DAOS_INSTALL_DIR"
@@ -218,27 +212,33 @@ if [[ ! -d "$DAOS_VENV_DIR" ]] ; then
 fi
 
 if [[ ! -f "$DAOS_VENV_DIR/bin/activate" ]] ; then
-	info "Installing avocado"
+	info "Creating DAOS python virtualenv $DAOS_VENV_DIR"
 	run $PYTHON_EXE -m venv "$DAOS_VENV_DIR"
-	source "$DAOS_VENV_DIR/bin/activate"
-	run $PIP_EXE install --upgrade pip
-	run $PIP_EXE install -r "$DAOS_SOURCE_DIR/requirements.txt"
-	deactivate
 fi
+info "Installing DAOS python virtualenv $DAOS_VENV_DIR"
+source "$DAOS_VENV_DIR/bin/activate"
+run $PIP_EXE install --upgrade pip
+run $PIP_EXE install -r "$DAOS_SOURCE_DIR/requirements.txt"
 
 SCONS_OPTS="--directory="$DAOS_SOURCE_DIR" --jobs=$JOBS_NB"
-case $CMD in
-	install) SCONS_OPTS="$SCONS_OPTS --build-deps=yes" ;;
-	update) SCONS_OPTS="$SCONS_OPTS --build-deps=no" ;;
-	*) fatal "Unexpected case" ;;
-esac
 
-info "Building and installing DAOS from source tree $PWD to $DAOS_INSTALL_DIR"
+if "$DAOS_BUILD_DEPS" ; then
+	info "Building DAOS dependencies from source tree $DAOS_SOURCE_DIR"
+	if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" $SCONS_OPTS --build-deps=only ; then
+		fatal "DAOS dependencies could not be properly build"
+	fi
+fi
+
+info "Building DAOS from source tree $DAOS_SOURCE_DIR"
+if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" $SCONS_OPTS --build-deps=no ; then
+	fatal "DAOS could not be properly build"
+fi
+
+info "Installing DAOS to $DAOS_INSTALL_DIR"
 source "$DAOS_VENV_DIR/bin/activate"
 if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
-	fatal "DAOS could not be properly build and/or installed"
+	fatal "DAOS could not be properly installed"
 fi
-deactivate
 
 info "Finalizing install of ftest setup scripts"
 for file_path in "$DAOS_SOURCE_DIR"/.build_vars*
@@ -601,6 +601,8 @@ if [[ ${FIO_PATH:+x} ]] ; then
 	popd > /dev/null
 fi
 
+deactivate
+
 info "Updating dynamic linker configuration"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
 run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST rm -f /etc/ld.so.cache

From 97ec61dff6c3f387c9fb772912d38c8d9f2604a1 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 13/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Remove useless python related packages
- Remove useless option and add a comment
- Remove useless opts

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/templates/daos-make.sh.j2 | 7 ++++---
 utils/ansible/ftest/vars/Rocky8.yml           | 9 ---------
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index 5d63972d128..a3304494035 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -224,19 +224,20 @@ SCONS_OPTS="--directory="$DAOS_SOURCE_DIR" --jobs=$JOBS_NB"
 
 if "$DAOS_BUILD_DEPS" ; then
 	info "Building DAOS dependencies from source tree $DAOS_SOURCE_DIR"
-	if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" $SCONS_OPTS --build-deps=only ; then
+	if ! run "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" $SCONS_OPTS --build-deps=only ; then
 		fatal "DAOS dependencies could not be properly build"
 	fi
 fi
 
 info "Building DAOS from source tree $DAOS_SOURCE_DIR"
-if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" $SCONS_OPTS --build-deps=no ; then
+# NOTE Dependencies will not be build as 'no' is default value of the --build-deps option
+if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" $SCONS_OPTS ; then
 	fatal "DAOS could not be properly build"
 fi
 
 info "Installing DAOS to $DAOS_INSTALL_DIR"
 source "$DAOS_VENV_DIR/bin/activate"
-if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" install $SCONS_OPTS ; then
+if ! run "$SCONS_EXE" install $SCONS_OPTS ; then
 	fatal "DAOS could not be properly installed"
 fi
 
diff --git a/utils/ansible/ftest/vars/Rocky8.yml b/utils/ansible/ftest/vars/Rocky8.yml
index 3e03527ee4e..4ee3a5e2cf0 100644
--- a/utils/ansible/ftest/vars/Rocky8.yml
+++ b/utils/ansible/ftest/vars/Rocky8.yml
@@ -44,17 +44,8 @@ daos_dev_deps:
     - python3.11
     - python3.11-devel
     - python3-clustershell
-    - python3-defusedxml
-    - python3-distro
-    - python3-jira
-    - python3-junit_xml
     - python3-pip
-    - python3-pyelftools
-    - python3-scons
-    - python3-tabulate
     - python3-wheel
-    - python3-pyxattr
-    - python3-pyyaml
     - rpmdevtools
     - rpm-build
     - spdk-devel

From 560ff4c6d512bd333d3751930998ebec9cc2aa4e Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 14/17] DAOS-15603: Support of FIO with ansible ftest script

Fix regression with force install

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/templates/daos-make.sh.j2 | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index a3304494035..ea1d997afda 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -286,7 +286,7 @@ if [[ ${MPICH_PATH:+x} ]] ; then
 		fatal "Invalid MPICH directory: path=$MPICH_PATH"
 	fi
 
-	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
+	if "$FORCE_INSTALL" ; then
 		info "Removing old MPICH install"
 		if ! run rm -fr "$MPICH_PATH/build" "$MPICH_PATH/configure" ; then
 			fatal "Old install of MPICH could not be removed"
@@ -383,7 +383,7 @@ if [[ ${HDF5_PATH:+x} ]] ; then
 		fatal "Invalid HDF5 directory: path=$HDF5_PATH"
 	fi
 
-	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
+	if "$FORCE_INSTALL" ; then
 		info "Removing old HDF5 install"
 		if ! run rm -fr "$HDF5_PATH/build" "$HDF5_PATH/configure" ; then
 			fatal "Old install of HDF5 could not be removed"
@@ -426,7 +426,7 @@ if [[ ${VOL_DAOS_PATH:+x} ]] ; then
 		fatal "Invalid DAOS HDF5 Volume directory: path=$VOL_DAOS_PATH"
 	fi
 
-	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
+	if "$FORCE_INSTALL" ; then
 		info "Removing old DAOS HDF5 Volume install"
 		if ! run rm -fr "$VOL_DAOS_PATH/build" ; then
 			fatal "Old install of HDF5 could not be removed"
@@ -472,7 +472,7 @@ if [[ ${MPI_FILE_UTILS_PATH:+x} ]] ; then
 		fatal "Invalid MPI File Utils directory: path=$MPI_FILE_UTILS_PATH"
 	fi
 
-	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
+	if "$FORCE_INSTALL" ; then
 		info "Removing old MPI File Utils install"
 		if ! run rm -fr "$MPI_FILE_UTILS_PATH/deps" "$MPI_FILE_UTILS_PATH/build" "$MPI_FILE_UTILS_PATH/configure" ; then
 			fatal "Old install of MPI File Utils could not be removed"
@@ -543,7 +543,7 @@ if [[ ${IOR_PATH:+x} ]] ; then
 		fatal "Invalid IOR directory: path=$IOR_PATH"
 	fi
 
-	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
+	if "$FORCE_INSTALL" ; then
 		info "Removing old IOR install"
 		if ! run rm -fr "$IOR_PATH/build" "$IOR_PATH/configure" ; then
 			fatal "Old install of IOR could not be removed"
@@ -583,7 +583,7 @@ if [[ ${FIO_PATH:+x} ]] ; then
 		fatal "Invalid FIO: path=$FIO_PATH"
 	fi
 
-	if [[ "$CMD" == install ]] && "$FORCE_INSTALL" ; then
+	if "$FORCE_INSTALL" ; then
 		info "Removing old FIO install"
 		if ! run rm -fr "$FIO_PATH/build" ; then
 			fatal "Old install of FIO could not be removed"

From 03ae13204aacf1821a8cd75734207322c7ed3881 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 15/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Remove useless dependencies

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/vars/Rocky8.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/utils/ansible/ftest/vars/Rocky8.yml b/utils/ansible/ftest/vars/Rocky8.yml
index 4ee3a5e2cf0..4f15333022c 100644
--- a/utils/ansible/ftest/vars/Rocky8.yml
+++ b/utils/ansible/ftest/vars/Rocky8.yml
@@ -38,8 +38,6 @@ daos_dev_deps:
     - libcmocka-devel
     - libevent-devel
     - libyaml-devel
-    - meson
-    - ninja-build
     - openssl-devel
     - python3.11
     - python3.11-devel

From 7d876bc371e97e6a19ce79e006142022e2fff1a6 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 16/17] DAOS-15603: Support of FIO with ansible ftest script

Integrate reviewers comments:
- Fix invalid missing exit 1

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/file/configure-nofile.sh | 1 +
 utils/ansible/ftest/file/enable-coredumps.sh | 1 +
 2 files changed, 2 insertions(+)

diff --git a/utils/ansible/ftest/file/configure-nofile.sh b/utils/ansible/ftest/file/configure-nofile.sh
index 1ca680d9350..2426a166bd1 100644
--- a/utils/ansible/ftest/file/configure-nofile.sh
+++ b/utils/ansible/ftest/file/configure-nofile.sh
@@ -5,6 +5,7 @@ set -u -e -o pipefail
 
 if [[ $(id -u) -ne 0 ]] ; then
 	echo "[ERROR] Could only be used by root"
+	exit 1
 fi
 
 sed -i -E -e "/DefaultLimitNOFILE/s/.*/DefaultLimitNOFILE=infinity:infinity/" /etc/systemd/system.conf
diff --git a/utils/ansible/ftest/file/enable-coredumps.sh b/utils/ansible/ftest/file/enable-coredumps.sh
index 6746ea718b2..0f703919d0d 100644
--- a/utils/ansible/ftest/file/enable-coredumps.sh
+++ b/utils/ansible/ftest/file/enable-coredumps.sh
@@ -5,6 +5,7 @@ set -u -e -o pipefail
 
 if [[ $(id -u) -ne 0 ]] ; then
 	echo "[ERROR] Could only be used by root"
+	exit 1
 fi
 
 sed -i -E -e "/DefaultLimitCORE/s/.*/DefaultLimitCORE=infinity:infinity/" /etc/systemd/system.conf

From a91ce3deb8693767bfa4001c5e2b7b9df3abf3d3 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <cedric.koch.hofer@gmail.com>
Date: Tue, 9 Apr 2024 11:19:13 +0000
Subject: [PATCH 17/17] DAOS-15603: Support of FIO with ansible ftest script

Build and install FIO man pages

Doc-only: true
Required-githooks: true

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 utils/ansible/ftest/templates/daos-make.sh.j2 | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2
index ea1d997afda..e1bde6a59b5 100644
--- a/utils/ansible/ftest/templates/daos-make.sh.j2
+++ b/utils/ansible/ftest/templates/daos-make.sh.j2
@@ -241,6 +241,8 @@ if ! run "$SCONS_EXE" install $SCONS_OPTS ; then
 	fatal "DAOS could not be properly installed"
 fi
 
+deactivate
+
 info "Finalizing install of ftest setup scripts"
 for file_path in "$DAOS_SOURCE_DIR"/.build_vars*
 do
@@ -600,9 +602,27 @@ if [[ ${FIO_PATH:+x} ]] ; then
 	run env CFLAGS="-g -O0" make -j $(nproc) install
 
 	popd > /dev/null
-fi
 
-deactivate
+	pushd "$FIO_PATH/doc" > /dev/null
+
+	FIO_VENV_DIR="$FIO_PATH/build/virtualenvs"
+	if [[ ! -f "$FIO_VENV_DIR/bin/activate" ]] ; then
+		info "Creating FIO python virtualenv $FIO_VENV_DIR"
+		run $PYTHON_EXE -m venv "$FIO_VENV_DIR"
+	fi
+
+	info "Installing FIO python virtualenv $FIO_VENV_DIR"
+	source "$FIO_VENV_DIR/bin/activate"
+	run $PIP_EXE install --upgrade pip
+	run $PIP_EXE install sphinx
+
+	run make man
+	run install -v output/man/fio.1 $DAOS_INSTALL_DIR/man/man1/fio.1
+
+	deactivate
+
+	popd
+fi
 
 info "Updating dynamic linker configuration"
 {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}