diff --git a/.dockerignore b/.dockerignore index a57616411db..8d2f7e4a386 100644 --- a/.dockerignore +++ b/.dockerignore @@ -30,6 +30,7 @@ !requirements-build.txt !requirements-utest.txt !requirements-ftest.txt +!requirements-code-coverage.txt # Now disallow extra files which may be present but are not required. # Include the rdb repo as well. diff --git a/Jenkinsfile b/Jenkinsfile index cf92efd1610..6a8d09be53f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -218,9 +218,22 @@ Boolean skip_build_stage(String distro='', String compiler='gcc') { return false } +Boolean is_code_coverage() { + if (startedByTimer()) { + return true + } + return paramsValue('CI_CODE_COVERAGE', false) +} + pipeline { agent { label 'lightweight' } + triggers { + // Generate a code coverage report each Sunday + /* groovylint-disable-next-line AddEmptyString */ + cron(env.BRANCH_NAME == 'master' ? 'TZ=UTC\n0 6 * * 0' : '') + } + environment { GITHUB_USER = credentials('daos-jenkins-review-posting') SSH_KEY_ARGS = '-ici_key' @@ -230,6 +243,7 @@ pipeline { REPO_FILE_URL = repoFileUrl(env.REPO_FILE_URL) SCONS_FAULTS_ARGS = sconsArgs() HTTPS_PROXY = '' + PYTHON_VERSION = '3.11' } options { @@ -305,6 +319,9 @@ pipeline { booleanParam(name: 'CI_leap15_NOBUILD', defaultValue: false, description: 'Do not build sources and RPMs on Leap 15') + booleanParam(name: 'CI_CODE_COVERAGE', + defaultValue: false, + description: 'Run with code coverage analysis') booleanParam(name: 'CI_ALLOW_UNSTABLE_TEST', defaultValue: false, description: 'Continue testing if a previous stage is Unstable') @@ -552,7 +569,8 @@ pipeline { build_deps: 'no', stash_opt: true, scons_args: sconsArgs() + - ' PREFIX=/opt/daos TARGET_TYPE=release')) + ' PREFIX=/opt/daos TARGET_TYPE=release', + code_coverage: is_code_coverage())) sh label: 'Generate RPMs', script: './ci/rpm/gen_rpms.sh el8 "' + env.DAOS_RELVAL + '"' } @@ -604,7 +622,8 @@ pipeline { build_deps: 'no', stash_opt: true, scons_args: sconsArgs() + - ' PREFIX=/opt/daos TARGET_TYPE=release')) + ' PREFIX=/opt/daos TARGET_TYPE=release', + code_coverage: is_code_coverage())) sh label: 'Generate RPMs', script: './ci/rpm/gen_rpms.sh el9 "' + env.DAOS_RELVAL + '"' } @@ -651,9 +670,10 @@ pipeline { script: './ci/rpm/build_deps.sh' job_step_update( sconsBuild(parallel_build: true, - scons_args: sconsFaultsArgs() + - ' PREFIX=/opt/daos TARGET_TYPE=release', - build_deps: 'yes')) + scons_args: sconsFaultsArgs() + + ' PREFIX=/opt/daos TARGET_TYPE=release', + code_coverage: is_code_coverage(), + build_deps: 'yes')) sh label: 'Generate RPMs', script: './ci/rpm/gen_rpms.sh suse.lp155 "' + env.DAOS_RELVAL + '"' } @@ -752,7 +772,7 @@ pipeline { } steps { job_step_update( - unitTest(timeout_time: 60, + unitTest(timeout_time: 120, unstash_opt: true, inst_repos: daosRepos(), inst_rpms: unitPackages())) @@ -767,7 +787,7 @@ pipeline { stage('NLT on EL 8.8') { when { beforeAgent true - expression { params.CI_NLT_TEST && !skipStage() } + expression { params.CI_NLT_TEST && !skipStage() && !is_code_coverage() } } agent { label params.CI_NLT_1_LABEL @@ -814,7 +834,7 @@ pipeline { } steps { job_step_update( - unitTest(timeout_time: 160, + unitTest(timeout_time: 250, unstash_opt: true, ignore_failure: true, inst_repos: daosRepos(), @@ -822,8 +842,7 @@ pipeline { } post { always { - unitTestPost artifacts: ['unit_test_memcheck_logs.tar.gz', - 'unit_test_memcheck_logs/**/*.log'], + unitTestPost artifacts: ['unit_test_memcheck_logs/'], valgrind_stash: 'el8-gcc-unit-memcheck' job_status_update() } @@ -839,7 +858,7 @@ pipeline { } steps { job_step_update( - unitTest(timeout_time: 180, + unitTest(timeout_time: 450, unstash_opt: true, ignore_failure: true, inst_repos: daosRepos(), @@ -847,8 +866,7 @@ pipeline { } post { always { - unitTestPost artifacts: ['unit_test_memcheck_bdev_logs.tar.gz', - 'unit_test_memcheck_bdev_logs/**/*.log'], + unitTestPost artifacts: ['unit_test_memcheck_bdev_logs/'], valgrind_stash: 'el8-gcc-unit-memcheck-bdev' job_status_update() } @@ -859,9 +877,9 @@ pipeline { stage('Test') { when { beforeAgent true - //expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) && !skipStage() } + // expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) && !skipStage() } // Above not working, always skipping functional VM tests. - expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) } + expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) && !is_code_coverage() } } parallel { stage('Functional on EL 8.8 with Valgrind') { @@ -1128,7 +1146,7 @@ pipeline { stage('Test Hardware') { when { beforeAgent true - expression { !paramsValue('CI_FUNCTIONAL_HARDWARE_TEST_SKIP', false) && !skipStage() } + expression { !paramsValue('CI_FUNCTIONAL_HARDWARE_TEST_SKIP', false) && !skipStage() && !is_code_coverage() } } steps { script { @@ -1237,6 +1255,46 @@ pipeline { } } } // stage('Test Hardware') + stage('Test Summary') { + when { + beforeAgent true + expression { true } + } + parallel { + stage('Code Coverage Report') { + when { + beforeAgent true + expression { params.CI_CODE_COVERAGE } + } + agent { + dockerfile { + filename 'utils/docker/Dockerfile.el.8' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs(add_repos: false) + + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + } + } + steps { + job_step_update( + runScriptWithStashes( + stashes: ['code_coverage_Unit_Test_on_EL_8.8', + 'code_coverage_Unit_Test_bdev_on_EL_8.8', + 'code_coverage_NLT_on_EL_8.8', + 'code_coverage_Unit_Test_with_memcheck_on_EL_8.8', + 'code_coverage_Unit_Test_bdev_with_memcheck_on_EL_8.8'], + script: 'ci/code_coverage_report.sh', + label: 'Code Coverage Report')) + } + post { + always { + archiveArtifacts artifacts: 'code_coverage_report/*', + allowEmptyArchive: false + job_status_update() + } + } + } // stage('Code Coverage Report') + } // parallel + } // stage('Test Summary') } // stages post { always { diff --git a/SConstruct b/SConstruct index c7092c51f79..933823c2719 100644 --- a/SConstruct +++ b/SConstruct @@ -93,6 +93,13 @@ def add_command_line_options(): use the specified relative sub-directory containing all \ dependencies as git submodules instead') + # generate code coverage + AddOption('--code-coverage', + action='store_true', + dest='code_coverage', + default=False, + help='enable code coverage analyze with gcov') + def parse_and_save_conf(env, opts_file): """Parse daos.conf @@ -469,6 +476,11 @@ def scons(): # This will add a final 'DEPS' value to opts but it will not be persistent. prereqs.run_build(opts) + have_hdf5 = False + if prereqs.check_component('hdf5'): + have_hdf5 = True + Export('have_hdf5') + if GetOption('build_deps') == 'only': prereqs.save_build_info() print('Exiting because --build-deps=only was set') diff --git a/ci/code_coverage_report.sh b/ci/code_coverage_report.sh new file mode 100755 index 00000000000..54f576beaf9 --- /dev/null +++ b/ci/code_coverage_report.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eux + +: "${PYTHON_VERSION:=3.11}" + +# Generate code coverage report +if [[ -n $(find . -name "code_coverage.json") ]]; then + "python${PYTHON_VERSION}" -m venv venv + # shellcheck disable=SC1091 + source venv/bin/activate + pip install --upgrade pip + pip install --requirement requirements-code-coverage.txt + + mkdir -p code_coverage_report + gcovr --add-tracefile "*/code_coverage/code_coverage.json" \ + --output code_coverage_report/code_coverage_report.html \ + --html-details --gcov-ignore-parse-errors +fi diff --git a/ci/provisioning/post_provision_config_nodes_EL.sh b/ci/provisioning/post_provision_config_nodes_EL.sh index 75e1d7934e3..f49f7f49b30 100644 --- a/ci/provisioning/post_provision_config_nodes_EL.sh +++ b/ci/provisioning/post_provision_config_nodes_EL.sh @@ -27,11 +27,11 @@ group_repo_post() { distro_custom() { # TODO: This code is not exiting on failure. + : "${PYTHON_VERSION:=3.11}" # Use a more recent python version for unit testing, this allows us to also test installing # pydaos into virtual environments. - dnf -y install python39 python39-devel - dnf -y install python3.11 python3.11-devel + dnf -y install "python${PYTHON_VERSION}" "python${PYTHON_VERSION}-devel" } install_mofed() { diff --git a/ci/provisioning/post_provision_config_nodes_LEAP.sh b/ci/provisioning/post_provision_config_nodes_LEAP.sh index 5a2a553e56f..d15554ad004 100644 --- a/ci/provisioning/post_provision_config_nodes_LEAP.sh +++ b/ci/provisioning/post_provision_config_nodes_LEAP.sh @@ -23,4 +23,9 @@ distro_custom() { sed -e '/MODULEPATH=/s/$/:\/usr\/share\/modules/' \ /etc/profile.d/lmod.sh; \ fi + + # Use a more recent python version for unit testing, this allows us to also test installing + # pydaos into virtual environments. + : "${PYTHON_VERSION:=}" + dnf -y install "python${PYTHON_VERSION//./}" "python${PYTHON_VERSION//./}-devel" } diff --git a/ci/unit/test_main_node.sh b/ci/unit/test_main_node.sh index 0afbf26fea6..29eb7714d1c 100755 --- a/ci/unit/test_main_node.sh +++ b/ci/unit/test_main_node.sh @@ -80,18 +80,34 @@ fi rm -rf "$test_log_dir" -# Use default python as that's where storage_estimator is installed. -python3 -m venv venv +: "${PYTHON_VERSION:=3.11}" +"python${PYTHON_VERSION}" -m venv venv # shellcheck disable=SC1091 source venv/bin/activate -# touch venv/pip.conf -# pip config set global.progress_bar off -# pip config set global.no_color true - +touch venv/pip.conf +pip config set global.progress_bar off +pip config set global.no_color true pip install --upgrade pip pip install --requirement requirements-utest.txt - pip install /opt/daos/lib/daos/python/ +WITH_CODE_COVERAGE="no" +if [[ -n $(find build -name "*.gcno" | xargs ls -la) ]]; then + WITH_CODE_COVERAGE="yes" +fi + HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" utils/run_utest.py $RUN_TEST_VALGRIND \ - --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" $SUDO_ARG + --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" $SUDO_ARG \ + --with_code_coverage="$WITH_CODE_COVERAGE" + +# Generate code coverage report if at least one gcda file was generated +if [ "$WITH_CODE_COVERAGE" = "yes" ]; then + pip install --requirement requirements-code-coverage.txt + find . -name "*.gcno" | xargs ls -la + find . -name "*.gcda" | xargs ls -la + find . -name "*.c" | xargs ls -la + find . -name "*.h" | xargs ls -la + mkdir -p "${test_log_dir}/code_coverage" + gcovr --json "${test_log_dir}/code_coverage/code_coverage.json" --gcov-ignore-parse-errors \ + --gcov-ignore-errors=no_working_dir_found +fi diff --git a/ci/unit/test_nlt.sh b/ci/unit/test_nlt.sh index a5f50545a15..482cdf235eb 100755 --- a/ci/unit/test_nlt.sh +++ b/ci/unit/test_nlt.sh @@ -10,8 +10,8 @@ rm -rf dnt.*.memcheck.xml vm_test/ NODE=${NODELIST%%,*} mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -# Copy over the install tree and some of the build tree. -rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils requirements-utest.txt jenkins@"$NODE":build/ +# Copy over the install tree and the build tree. +rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" . jenkins@"$NODE":build/ # shellcheck disable=SC2029 ssh -tt "$SSH_KEY_ARGS" jenkins@"$NODE" "DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \ diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index fa422586ad9..3138150d4e7 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -25,21 +25,35 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh" # --server-valgrind all # Use the latest version that CI has available. -python3.11 -m venv venv +: "${PYTHON_VERSION:=3.11}" +"python${PYTHON_VERSION}" -m venv venv # shellcheck disable=SC1091 source venv/bin/activate touch venv/pip.conf pip config set global.progress_bar off pip config set global.no_color true - pip install --upgrade pip pip install --requirement requirements-utest.txt - pip install /opt/daos/lib/daos/python/ # set high open file limit in the shell to avoid extra warning sudo prlimit --nofile=1024:262144 --pid $$ prlimit -n -HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" ./utils/node_local_test.py --max-log-size 1950MiB \ - --dfuse-dir /localhome/jenkins/ --log-usage-save nltir.xml --log-usage-export nltr.json all +cwd="$(pwd)" +# export GCOV_PREFIX="${cwd}" +# export GCOV_PREFIX_STRIP="7" +# HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" ./utils/node_local_test.py --max-log-size 1950MiB \ +# --dfuse-dir /localhome/jenkins/ --log-usage-save nltir.xml --log-usage-export nltr.json all + +GCOV_PREFIX="${cwd}" GCOV_PREFIX_STRIP="7" HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" \ + ./utils/node_local_test.py --max-log-size 1950MiB --dfuse-dir /localhome/jenkins/ \ + --log-usage-save nltir.xml --log-usage-export nltr.json all + +# Generate code coverage report if at least one gcda file was generated +if [[ -n $(find build -name "*.gcda") ]]; then + pip install --requirement requirements-code-coverage.txt + + mkdir -p /tmp/code_coverage + gcovr --json /tmp/code_coverage/code_coverage.json --gcov-ignore-parse-errors +fi diff --git a/ci/unit/test_nlt_post.sh b/ci/unit/test_nlt_post.sh index c46a63dac2f..23ef28e4af4 100755 --- a/ci/unit/test_nlt_post.sh +++ b/ci/unit/test_nlt_post.sh @@ -15,7 +15,7 @@ mkdir nlt_logs # standard wildcards. rsync -v -dprt -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":/tmp/ \ --filter="include dnt*.log" --filter="include dnt*.log.bz2" \ - --filter="include dnt_fi_*_logs" \ + --filter="include dnt_fi_*_logs" --filter="include code_coverage*" \ --filter="exclude *" nlt_logs/ rsync -v -dpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ \ diff --git a/ci/unit/test_post_always.sh b/ci/unit/test_post_always.sh index 5d48ab5482d..b185aeacc9b 100755 --- a/ci/unit/test_post_always.sh +++ b/ci/unit/test_post_always.sh @@ -39,11 +39,11 @@ esac mkdir -p "$test_log_dir" -rsync -v -rlpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ . +rsync -v -rlpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ --exclude="venv" . # Copy any log files. Use rsync filters here to allow us to specify # all files we want to copy, as it's much more flexible than using # standard wildcards. rsync -v -dpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":/tmp/ \ - --filter="include daos*.log" --filter="include test.cov" \ + --filter="include daos*.log" --filter="include code_coverage/*" \ --filter="exclude *" "$test_log_dir/" diff --git a/requirements-code-coverage.txt b/requirements-code-coverage.txt new file mode 100644 index 00000000000..4810efd8455 --- /dev/null +++ b/requirements-code-coverage.txt @@ -0,0 +1,3 @@ +gcovr +junit-xml +pyyaml diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index da2518a9928..db9ea379361 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -240,6 +240,8 @@ def define_common(reqs): reqs.define('boost', headers=['boost/preprocessor.hpp'], package='boost-python36-devel') + reqs.define('hdf5', headers=['hdf5.h'], package='hdf5-devel') + reqs.define('yaml', headers=['yaml.h'], package='libyaml-devel') reqs.define('event', libs=['event'], package='libevent-devel') diff --git a/site_scons/site_tools/compiler_setup.py b/site_scons/site_tools/compiler_setup.py index 812ef0632dd..5e086045b4d 100644 --- a/site_scons/site_tools/compiler_setup.py +++ b/site_scons/site_tools/compiler_setup.py @@ -35,6 +35,7 @@ def _base_setup(env): Include all our preferred compile options for the chosen compiler and build type. """ + # pylint: disable=too-many-branches if GetOption('help') or GetOption('clean'): return @@ -126,6 +127,11 @@ def _base_setup(env): env['BSETUP'] = compiler + if GetOption('code_coverage'): + env.AppendUnique(CXXFLAGS=['-fprofile-arcs', '-ftest-coverage']) + env.AppendUnique(CCFLAGS=['-fprofile-arcs', '-ftest-coverage']) + env.AppendUnique(CGO_CFLAGS=['-fprofile-arcs', '-ftest-coverage']) + def _check_flag_helper(context, compiler, ext, flag): """Helper function to allow checking for compiler flags""" diff --git a/site_scons/site_tools/daos_builder.py b/site_scons/site_tools/daos_builder.py index c3eb6d0c62d..848fa053b4d 100644 --- a/site_scons/site_tools/daos_builder.py +++ b/site_scons/site_tools/daos_builder.py @@ -121,6 +121,10 @@ def _add_lib(libtype, libname, target): def _run_command(env, target, sources, daos_libs, command): """Run Command builder""" + if GetOption("code_coverage"): + if 'gcov' not in daos_libs: + daos_libs += ['gcov'] + env.AppendENVPath('CGO_LDFLAGS', '-lgcov', sep=' ') static_deps, shared_deps = _known_deps(env, LIBS=daos_libs) result = env.Command(target, sources + static_deps + shared_deps, command) return result @@ -128,6 +132,7 @@ def _run_command(env, target, sources, daos_libs, command): def _static_library(env, *args, **kwargs): """Build SharedLibrary with relative RPATH""" + kwargs = _add_code_coverage(env, **kwargs) libname = _get_libname(*args, **kwargs) if 'hide_syms' in kwargs: # Allow for auto-hiding of symbols, used for the Interception library. There are multiple @@ -153,6 +158,7 @@ def _static_library(env, *args, **kwargs): def _library(env, *args, **kwargs): """Build SharedLibrary with relative RPATH""" denv = env.Clone() + kwargs = _add_code_coverage(denv, **kwargs) denv.Replace(RPATH=[]) _add_rpaths(denv, kwargs.get('install_off', '..'), False, False) lib = denv.SharedLibrary(*args, **kwargs) @@ -167,6 +173,7 @@ def _library(env, *args, **kwargs): def _program(env, *args, **kwargs): """Build Program with relative RPATH""" denv = env.Clone() + kwargs = _add_code_coverage(denv, **kwargs) denv.AppendUnique(LINKFLAGS=['-pie']) denv.Replace(RPATH=[]) _add_rpaths(denv, kwargs.get('install_off', '..'), False, True) @@ -180,6 +187,7 @@ def _program(env, *args, **kwargs): def _test_program(env, *args, **kwargs): """Build Program with fixed RPATH""" denv = env.Clone() + kwargs = _add_code_coverage(denv, **kwargs) denv.AppendUnique(LINKFLAGS=['-pie']) denv.Replace(RPATH=[]) _add_rpaths(denv, kwargs.get("install_off", None), False, True) @@ -248,6 +256,19 @@ def _print(msg): return None +def _add_code_coverage(env, **kwargs): + """Add library for code coverage""" + if GetOption("code_coverage"): + if 'LIBS' in kwargs: + if isinstance(kwargs['LIBS'], str): + kwargs['LIBS'] = [kwargs['LIBS']] + if 'gcov' not in kwargs['LIBS']: + kwargs['LIBS'].append('gcov') + else: + env.AppendUnique(LIBS=['gcov']) + return kwargs + + def generate(env): """Add daos specific methods to environment""" env.AddMethod(_add_build_rpath, 'd_add_build_rpath') diff --git a/src/client/serialize/SConscript b/src/client/serialize/SConscript index 83b077d7867..7436f88514c 100644 --- a/src/client/serialize/SConscript +++ b/src/client/serialize/SConscript @@ -4,18 +4,11 @@ def scons(): """Execute build""" - Import('env') + Import('env', 'have_hdf5') denv = env.Clone() - have_hdf5 = True - # If hdf5 installed build DAOS serialization library - conf = Configure(denv) - if not conf.CheckLibWithHeader('hdf5', 'hdf5.h', 'c'): - have_hdf5 = False - conf.Finish() - libraries = ['daos', 'hdf5'] src = ['daos_serialize.c'] diff --git a/src/control/run_go_tests.sh b/src/control/run_go_tests.sh index 5f727578283..4efc46c517b 100755 --- a/src/control/run_go_tests.sh +++ b/src/control/run_go_tests.sh @@ -96,6 +96,13 @@ function setup_environment() CGO_LDFLAGS+="${SL_SPDK_PREFIX+ -L${SL_SPDK_PREFIX}/lib64/daos_srv}" [[ -z "${SL_SPDK_PREFIX:-}" ]] && CGO_LDFLAGS+=" -L /usr/lib64/daos_srv" CGO_LDFLAGS+="${SL_OFI_PREFIX+ -L${SL_OFI_PREFIX}/lib}" + + build_dir="$(dirname "$build_source")" + if [[ -n $(find "$build_dir" -name "*.gcno") ]]; then + echo "including gcov flags to CGO_LDFLAGS .." + CGO_LDFLAGS+=" -lgcov" + fi + CGO_CFLAGS=${SL_PREFIX+-I${SL_PREFIX}/include} CGO_CFLAGS+="${SL_MERCURY_PREFIX+ -I${SL_MERCURY_PREFIX}/include}" CGO_CFLAGS+="${SL_SPDK_PREFIX+ -I${SL_SPDK_PREFIX}/include/daos_srv}" @@ -103,11 +110,12 @@ function setup_environment() CGO_CFLAGS+="${SL_OFI_PREFIX+ -I${SL_OFI_PREFIX}/include}" CGO_CFLAGS+="${SL_ARGOBOTS_PREFIX+ -I${SL_ARGOBOTS_PREFIX}/include}" - src_include="$(dirname "$build_source")/src/include" + src_include="$build_dir/src/include" if [ -d "$src_include" ]; then echo "including path \"${src_include}\" in CGO_CFLAGS" CGO_CFLAGS+=" -I${src_include}" fi + export CGO_CFLAGS LD_LIBRARY_PATH CGO_LDFLAGS } diff --git a/utils/docker/Dockerfile.el.8 b/utils/docker/Dockerfile.el.8 index bf80f3fc122..e36c079258f 100644 --- a/utils/docker/Dockerfile.el.8 +++ b/utils/docker/Dockerfile.el.8 @@ -70,14 +70,16 @@ RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/ USER daos_server:daos_server # Setup a python venv so that python packages can be installed locally. -RUN python3 -m venv /home/daos/venv +ARG PYTHON_VERSION=3.11 +RUN python${PYTHON_VERSION} -m venv /home/daos/venv ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. COPY requirements-build.txt requirements-utest.txt ./ -RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +RUN source /home/daos/venv/bin/activate && \ + pip --no-cache-dir install --upgrade pip && \ + pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index 82c5287ff26..fd65d077ff2 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -66,14 +66,16 @@ RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/ USER daos_server:daos_server # Setup a python venv so that python packages can be installed locally. -RUN python3 -m venv /home/daos/venv +ARG PYTHON_VERSION=3.11 +RUN python${PYTHON_VERSION} -m venv /home/daos/venv ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. COPY requirements-build.txt requirements-utest.txt ./ -RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +RUN source /home/daos/venv/bin/activate && \ + pip --no-cache-dir install --upgrade pip && \ + pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index 93b40c12630..6086a641e90 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -71,14 +71,16 @@ RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/ USER daos_server:daos_server # Setup a python venv so that python packages can be installed locally. -RUN python3 -m venv /home/daos/venv +ARG PYTHON_VERSION=3.11 +RUN python${PYTHON_VERSION} -m venv /home/daos/venv ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. COPY requirements-build.txt requirements-utest.txt ./ -RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +RUN source /home/daos/venv/bin/activate && \ + pip --no-cache-dir install --upgrade pip && \ + pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index 589c3f3dd9d..effa2dd64d6 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -64,14 +64,16 @@ RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/ USER daos_server:daos_server # Setup a python venv so that python packages can be installed locally. -RUN python3 -m venv /home/daos/venv +ARG PYTHON_VERSION=3 +RUN python${PYTHON_VERSION} -m venv /home/daos/venv ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. COPY requirements-build.txt requirements-utest.txt ./ -RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt +RUN . /home/daos/venv/bin/activate && \ + pip --no-cache-dir install --upgrade pip && \ + pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/run_utest.py b/utils/run_utest.py index 39261be2a51..b87a5e13cef 100755 --- a/utils/run_utest.py +++ b/utils/run_utest.py @@ -544,6 +544,10 @@ def skip(self, path_info, config, args): print(f"Skipped suite {self.name}, running on GitHub Actions") raise SuiteSkipped() + if args.with_code_coverage == "yes" and config.get("skip_with_code_coverage", False): + print(f"Skipped suite {self.name}, code coverage not supported") + raise SuiteSkipped() + return False def needs_aio(self): @@ -641,6 +645,8 @@ def get_args(): help="Device to use for AIO, will create file by default") parser.add_argument('--log_dir', default="/tmp/daos_utest", help="Path to store test logs") + parser.add_argument('--with_code_coverage', choices=['yes', 'no'], default='no', + help='Tests compiled with code coverage') return parser.parse_args() diff --git a/utils/scripts/helpers/repo-helper-el8.sh b/utils/scripts/helpers/repo-helper-el8.sh index 3572699d634..d11a6089424 100755 --- a/utils/scripts/helpers/repo-helper-el8.sh +++ b/utils/scripts/helpers/repo-helper-el8.sh @@ -134,5 +134,7 @@ if [ -n "$REPO_FILE_URL" ]; then [global] trusted-host = ${trusted_host} index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple + progress_bar = off + no_color = true EOF fi diff --git a/utils/scripts/helpers/repo-helper-el9.sh b/utils/scripts/helpers/repo-helper-el9.sh index b595d5d5c15..02811f7c715 100644 --- a/utils/scripts/helpers/repo-helper-el9.sh +++ b/utils/scripts/helpers/repo-helper-el9.sh @@ -130,5 +130,7 @@ if [ -n "$REPO_FILE_URL" ]; then [global] trusted-host = ${trusted_host} index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple + progress_bar = off + no_color = true EOF fi diff --git a/utils/scripts/helpers/repo-helper-leap15.sh b/utils/scripts/helpers/repo-helper-leap15.sh index 98ced4e82ea..eaf0cb7c29b 100755 --- a/utils/scripts/helpers/repo-helper-leap15.sh +++ b/utils/scripts/helpers/repo-helper-leap15.sh @@ -178,5 +178,7 @@ if [ -n "$REPO_FILE_URL" ]; then [global] trusted-host = ${trusted_host} index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple + progress_bar = off + no_color = true EOF fi diff --git a/utils/scripts/helpers/repo-helper-ubuntu.sh b/utils/scripts/helpers/repo-helper-ubuntu.sh index c7738a982de..3557a5306fd 100644 --- a/utils/scripts/helpers/repo-helper-ubuntu.sh +++ b/utils/scripts/helpers/repo-helper-ubuntu.sh @@ -115,5 +115,7 @@ if [ -n "$REPO_FILE_URL" ]; then [global] trusted-host = ${trusted_host} index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple + progress_bar = off + no_color = true EOF fi diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index 670ef8f6eef..eafea251761 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -15,6 +15,8 @@ arch=$(uname -m) dnf_install_args="${1:-}" +: "${PYTHON_VERSION:=3.11}" + # shellcheck disable=SC2086 dnf --nodocs install ${dnf_install_args} \ boost-python3-devel \ @@ -72,8 +74,8 @@ dnf --nodocs install ${dnf_install_args} \ pciutils \ pciutils-devel \ protobuf-c-devel \ - python3-devel \ - python3-pip \ + python${PYTHON_VERSION}-devel \ + python${PYTHON_VERSION}-pip \ rpm-build \ sg3_utils \ squashfs-tools \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index a9234ff7cba..c4c0a5ad69f 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -13,6 +13,8 @@ set -e dnf_install_args="${1:-}" +: "${PYTHON_VERSION:=3.11}" + # shellcheck disable=SC2086 dnf --nodocs install ${dnf_install_args} \ boost-python3-devel \ @@ -73,8 +75,8 @@ dnf --nodocs install ${dnf_install_args} \ pciutils \ pciutils-devel \ protobuf-c-devel \ - python3-devel \ - python3-pip \ + python${PYTHON_VERSION}-devel \ + python${PYTHON_VERSION}-pip \ rpm-build \ sg3_utils \ squashfs-tools \ diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index 5029eb1000a..724e630ffbe 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -15,6 +15,8 @@ arch=$(uname -i) dnf_install_args="${1:-}" +: "${PYTHON_VERSION:=3.11}" + # shellcheck disable=SC2086 dnf --nodocs install ${dnf_install_args} \ boost-devel \ @@ -69,7 +71,7 @@ dnf --nodocs install ${dnf_install_args} \ patchelf \ pciutils \ pciutils-devel \ - python3-devel \ + python${PYTHON_VERSION//./}-devel \ rpm-build \ scons \ sg3_utils \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index 0db922daeb7..0cc903a194e 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -17,6 +17,8 @@ arch=$(uname -i) apt_get_install_args="${1:-}" +: "${PYTHON_VERSION:=3}" + # shellcheck disable=SC2086 apt-get install ${apt_get_install_args} \ autoconf \ @@ -62,8 +64,8 @@ apt-get install ${apt_get_install_args} \ pciutils \ pkg-config \ ruby \ - python3-dev \ - python3-venv \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-venv \ sudo \ uuid-dev \ valgrind \ diff --git a/utils/utest.yaml b/utils/utest.yaml index 8710a1bae46..aa7095a2046 100644 --- a/utils/utest.yaml +++ b/utils/utest.yaml @@ -208,6 +208,7 @@ - name: Argobot stack overflow testing base: "BUILD_DIR" memcheck: False + skip_with_code_coverage: true tests: - cmd: ["src/engine/tests/abt_stack", "--check-overflow"] env_vars: