DAOS-17259 test: Adding gcov to daos build

.dockerignore

@@ -30,6 +30,7 @@
 !requirements-build.txt
 !requirements-utest.txt
 !requirements-ftest.txt
+!requirements-code-coverage.txt
 
 # Now disallow extra files which may be present but are not required.
 # Include the rdb repo as well.

Jenkinsfile

@@ -218,9 +218,22 @@ Boolean skip_build_stage(String distro='', String compiler='gcc') {
     return false
 }
 
+Boolean is_code_coverage() {
+    if (startedByTimer()) {
+        return true
+    }
+    return paramsValue('CI_CODE_COVERAGE', false)
+}
+
 pipeline {
     agent { label 'lightweight' }
 
+    triggers {
+        // Generate a code coverage report each Sunday
+        /* groovylint-disable-next-line AddEmptyString */
+        cron(env.BRANCH_NAME == 'master' ? 'TZ=UTC\n0 6 * * 0' : '')
+    }
+
     environment {
         GITHUB_USER = credentials('daos-jenkins-review-posting')
         SSH_KEY_ARGS = '-ici_key'
@@ -230,6 +243,7 @@ pipeline {
         REPO_FILE_URL = repoFileUrl(env.REPO_FILE_URL)
         SCONS_FAULTS_ARGS = sconsArgs()
         HTTPS_PROXY = ''
+        PYTHON_VERSION = '3.11'
     }
 
     options {
@@ -305,6 +319,9 @@ pipeline {
         booleanParam(name: 'CI_leap15_NOBUILD',
                      defaultValue: false,
                      description: 'Do not build sources and RPMs on Leap 15')
+        booleanParam(name: 'CI_CODE_COVERAGE',
+                     defaultValue: false,
+                     description: 'Run with code coverage analysis')
         booleanParam(name: 'CI_ALLOW_UNSTABLE_TEST',
                      defaultValue: false,
                      description: 'Continue testing if a previous stage is Unstable')
@@ -552,7 +569,8 @@ pipeline {
                                         build_deps: 'no',
                                         stash_opt: true,
                                         scons_args: sconsArgs() +
-                                                    ' PREFIX=/opt/daos TARGET_TYPE=release'))
+                                                    ' PREFIX=/opt/daos TARGET_TYPE=release',
+                                        code_coverage: is_code_coverage()))
                             sh label: 'Generate RPMs',
                                 script: './ci/rpm/gen_rpms.sh el8 "' + env.DAOS_RELVAL + '"'
                         }
@@ -604,7 +622,8 @@ pipeline {
                                            build_deps: 'no',
                                            stash_opt: true,
                                            scons_args: sconsArgs() +
-                                                      ' PREFIX=/opt/daos TARGET_TYPE=release'))
+                                                      ' PREFIX=/opt/daos TARGET_TYPE=release',
+                                           code_coverage: is_code_coverage()))
                             sh label: 'Generate RPMs',
                                 script: './ci/rpm/gen_rpms.sh el9 "' + env.DAOS_RELVAL + '"'
                         }
@@ -651,9 +670,10 @@ pipeline {
                                 script: './ci/rpm/build_deps.sh'
                             job_step_update(
                                 sconsBuild(parallel_build: true,
-                                scons_args: sconsFaultsArgs() +
-                                ' PREFIX=/opt/daos TARGET_TYPE=release',
-                                build_deps: 'yes'))
+                                           scons_args: sconsFaultsArgs() +
+                                                       ' PREFIX=/opt/daos TARGET_TYPE=release',
+                                           code_coverage: is_code_coverage(),
+                                           build_deps: 'yes'))
                             sh label: 'Generate RPMs',
                                 script: './ci/rpm/gen_rpms.sh suse.lp155 "' + env.DAOS_RELVAL + '"'
                         }
@@ -752,7 +772,7 @@ pipeline {
                     }
                     steps {
                         job_step_update(
-                            unitTest(timeout_time: 60,
+                            unitTest(timeout_time: 120,
                                      unstash_opt: true,
                                      inst_repos: daosRepos(),
                                      inst_rpms: unitPackages()))
@@ -767,7 +787,7 @@ pipeline {
                 stage('NLT on EL 8.8') {
                     when {
                         beforeAgent true
-                        expression { params.CI_NLT_TEST && !skipStage() }
+                        expression { params.CI_NLT_TEST && !skipStage() && !is_code_coverage() }
                     }
                     agent {
                         label params.CI_NLT_1_LABEL
@@ -814,16 +834,15 @@ pipeline {
                     }
                     steps {
                         job_step_update(
-                            unitTest(timeout_time: 160,
+                            unitTest(timeout_time: 250,
                                      unstash_opt: true,
                                      ignore_failure: true,
                                      inst_repos: daosRepos(),
                                      inst_rpms: unitPackages()))
                     }
                     post {
                         always {
-                            unitTestPost artifacts: ['unit_test_memcheck_logs.tar.gz',
-                                                     'unit_test_memcheck_logs/**/*.log'],
+                            unitTestPost artifacts: ['unit_test_memcheck_logs/'],
                                          valgrind_stash: 'el8-gcc-unit-memcheck'
                             job_status_update()
                         }
@@ -839,16 +858,15 @@ pipeline {
                     }
                     steps {
                         job_step_update(
-                            unitTest(timeout_time: 180,
+                            unitTest(timeout_time: 450,
                                      unstash_opt: true,
                                      ignore_failure: true,
                                      inst_repos: daosRepos(),
                                      inst_rpms: unitPackages()))
                     }
                     post {
                         always {
-                            unitTestPost artifacts: ['unit_test_memcheck_bdev_logs.tar.gz',
-                                                     'unit_test_memcheck_bdev_logs/**/*.log'],
+                            unitTestPost artifacts: ['unit_test_memcheck_bdev_logs/'],
                                          valgrind_stash: 'el8-gcc-unit-memcheck-bdev'
                             job_status_update()
                         }
@@ -859,9 +877,9 @@ pipeline {
         stage('Test') {
             when {
                 beforeAgent true
-                //expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false)  && !skipStage() }
+                // expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false)  && !skipStage() }
                 // Above not working, always skipping functional VM tests.
-                expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) }
+                expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) && !is_code_coverage() }
             }
             parallel {
                 stage('Functional on EL 8.8 with Valgrind') {
@@ -1128,7 +1146,7 @@ pipeline {
         stage('Test Hardware') {
             when {
                 beforeAgent true
-                expression { !paramsValue('CI_FUNCTIONAL_HARDWARE_TEST_SKIP', false)  && !skipStage() }
+                expression { !paramsValue('CI_FUNCTIONAL_HARDWARE_TEST_SKIP', false)  && !skipStage() && !is_code_coverage() }
             }
             steps {
                 script {
@@ -1237,6 +1255,46 @@ pipeline {
                 }
             }
         } // stage('Test Hardware')
+        stage('Test Summary') {
+            when {
+                beforeAgent true
+                expression { true }
+            }
+            parallel {
+                stage('Code Coverage Report') {
+                    when {
+                        beforeAgent true
+                        expression { params.CI_CODE_COVERAGE }
+                    }
+                    agent {
+                        dockerfile {
+                            filename 'utils/docker/Dockerfile.el.8'
+                            label 'docker_runner'
+                            additionalBuildArgs dockerBuildArgs(add_repos: false) +
+                                                " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}"
+                        }
+                    }
+                    steps {
+                        job_step_update(
+                            runScriptWithStashes(
+                                stashes: ['code_coverage_Unit_Test_on_EL_8.8',
+                                          'code_coverage_Unit_Test_bdev_on_EL_8.8',
+                                          'code_coverage_NLT_on_EL_8.8',
+                                          'code_coverage_Unit_Test_with_memcheck_on_EL_8.8',
+                                          'code_coverage_Unit_Test_bdev_with_memcheck_on_EL_8.8'],
+                                script: 'ci/code_coverage_report.sh',
+                                label: 'Code Coverage Report'))
+                    }
+                    post {
+                        always {
+                            archiveArtifacts artifacts: 'code_coverage_report/*',
+                                             allowEmptyArchive: false
+                            job_status_update()
+                        }
+                    }
+                } // stage('Code Coverage Report')
+            } // parallel
+        } // stage('Test Summary')
     } // stages
     post {
         always {

SConstruct

@@ -93,6 +93,13 @@ def add_command_line_options():
                     use the specified relative sub-directory containing all \
                     dependencies as git submodules instead')
 
+    # generate code coverage
+    AddOption('--code-coverage',
+              action='store_true',
+              dest='code_coverage',
+              default=False,
+              help='enable code coverage analyze with gcov')
+
 
 def parse_and_save_conf(env, opts_file):
     """Parse daos.conf
@@ -469,6 +476,11 @@ def scons():
     # This will add a final 'DEPS' value to opts but it will not be persistent.
     prereqs.run_build(opts)
 
+    have_hdf5 = False
+    if prereqs.check_component('hdf5'):
+        have_hdf5 = True
+    Export('have_hdf5')
+
     if GetOption('build_deps') == 'only':
         prereqs.save_build_info()
         print('Exiting because --build-deps=only was set')

ci/code_coverage_report.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eux
+
+: "${PYTHON_VERSION:=3.11}"
+
+# Generate code coverage report
+if [[ -n $(find . -name "code_coverage.json") ]]; then
+    "python${PYTHON_VERSION}" -m venv venv
+    # shellcheck disable=SC1091
+    source venv/bin/activate
+    pip install --upgrade pip
+    pip install --requirement requirements-code-coverage.txt
+
+    mkdir -p code_coverage_report
+    gcovr --add-tracefile "*/code_coverage/code_coverage.json" \
+          --output code_coverage_report/code_coverage_report.html \
+          --html-details --gcov-ignore-parse-errors
+fi

ci/provisioning/post_provision_config_nodes_EL.sh

@@ -27,11 +27,11 @@ group_repo_post() {
 
 distro_custom() {
     # TODO: This code is not exiting on failure.
+    : "${PYTHON_VERSION:=3.11}"
 
     # Use a more recent python version for unit testing, this allows us to also test installing
     # pydaos into virtual environments.
-    dnf -y install python39 python39-devel
-    dnf -y install python3.11 python3.11-devel
+    dnf -y install "python${PYTHON_VERSION}" "python${PYTHON_VERSION}-devel"
 }
 
 install_mofed() {

ci/provisioning/post_provision_config_nodes_LEAP.sh

@@ -23,4 +23,9 @@ distro_custom() {
         sed -e '/MODULEPATH=/s/$/:\/usr\/share\/modules/'                     \
                /etc/profile.d/lmod.sh;                                        \
     fi
+
+    # Use a more recent python version for unit testing, this allows us to also test installing
+    # pydaos into virtual environments.
+    : "${PYTHON_VERSION:=}"
+    dnf -y install "python${PYTHON_VERSION//./}" "python${PYTHON_VERSION//./}-devel"
 }

ci/unit/test_main_node.sh

@@ -80,18 +80,34 @@
 
 rm -rf "$test_log_dir"
 
-# Use default python as that's where storage_estimator is installed.
-python3 -m venv venv
+: "${PYTHON_VERSION:=3.11}"
+"python${PYTHON_VERSION}" -m venv venv
 # shellcheck disable=SC1091
 source venv/bin/activate
-# touch venv/pip.conf
-# pip config set global.progress_bar off
-# pip config set global.no_color true
-
+touch venv/pip.conf
+pip config set global.progress_bar off
+pip config set global.no_color true
 pip install --upgrade pip
 pip install --requirement requirements-utest.txt
-
 pip install /opt/daos/lib/daos/python/
 
+WITH_CODE_COVERAGE="no"
+if [[ -n $(find build -name "*.gcno" | xargs ls -la) ]]; then
+    WITH_CODE_COVERAGE="yes"
+fi
+
 HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" utils/run_utest.py $RUN_TEST_VALGRIND \
-    --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" $SUDO_ARG
+    --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" $SUDO_ARG \
+    --with_code_coverage="$WITH_CODE_COVERAGE"
+
+# Generate code coverage report if at least one gcda file was generated
+if [ "$WITH_CODE_COVERAGE" = "yes" ]; then
+    pip install --requirement requirements-code-coverage.txt
+    find . -name "*.gcno" | xargs ls -la
+    find . -name "*.gcda" | xargs ls -la
+    find . -name "*.c" | xargs ls -la
+    find . -name "*.h" | xargs ls -la
+    mkdir -p "${test_log_dir}/code_coverage"
+    gcovr --json "${test_log_dir}/code_coverage/code_coverage.json" --gcov-ignore-parse-errors \
+        --gcov-ignore-errors=no_working_dir_found
+fi

ci/unit/test_nlt.sh

@@ -10,8 +10,8 @@ rm -rf dnt.*.memcheck.xml vm_test/
 NODE=${NODELIST%%,*}
 mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
-# Copy over the install tree and some of the build tree.
-rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils requirements-utest.txt jenkins@"$NODE":build/
+# Copy over the install tree and the build tree.
+rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" . jenkins@"$NODE":build/
 
 # shellcheck disable=SC2029
 ssh -tt "$SSH_KEY_ARGS" jenkins@"$NODE" "DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \

ci/unit/test_nlt_node.sh

@@ -25,21 +25,35 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh"
 #			   --server-valgrind all
 
 # Use the latest version that CI has available.
-python3.11 -m venv venv
+: "${PYTHON_VERSION:=3.11}"
+"python${PYTHON_VERSION}" -m venv venv
 # shellcheck disable=SC1091
 source venv/bin/activate
 touch venv/pip.conf
 pip config set global.progress_bar off
 pip config set global.no_color true
-
 pip install --upgrade pip
 pip install --requirement requirements-utest.txt
-
 pip install /opt/daos/lib/daos/python/
 
 # set high open file limit in the shell to avoid extra warning
 sudo prlimit --nofile=1024:262144 --pid $$
 prlimit -n
 
-HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" ./utils/node_local_test.py --max-log-size 1950MiB \
-    --dfuse-dir /localhome/jenkins/ --log-usage-save nltir.xml --log-usage-export nltr.json all
+cwd="$(pwd)"
+# export GCOV_PREFIX="${cwd}"
+# export GCOV_PREFIX_STRIP="7"
+# HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" ./utils/node_local_test.py --max-log-size 1950MiB \
+#    --dfuse-dir /localhome/jenkins/ --log-usage-save nltir.xml --log-usage-export nltr.json all
+
+GCOV_PREFIX="${cwd}" GCOV_PREFIX_STRIP="7" HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" \
+    ./utils/node_local_test.py --max-log-size 1950MiB --dfuse-dir /localhome/jenkins/ \
+    --log-usage-save nltir.xml --log-usage-export nltr.json all
+
+# Generate code coverage report if at least one gcda file was generated
+if [[ -n $(find build -name "*.gcda") ]]; then
+    pip install --requirement requirements-code-coverage.txt
+
+    mkdir -p /tmp/code_coverage
+    gcovr --json /tmp/code_coverage/code_coverage.json --gcov-ignore-parse-errors
+fi

ci/unit/test_nlt_post.sh

@@ -15,7 +15,7 @@ mkdir nlt_logs
 # standard wildcards.
 rsync -v -dprt -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":/tmp/ \
       --filter="include dnt*.log" --filter="include dnt*.log.bz2" \
-      --filter="include dnt_fi_*_logs" \
+      --filter="include dnt_fi_*_logs" --filter="include code_coverage*" \
       --filter="exclude *" nlt_logs/
 
 rsync -v -dpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ \

ci/unit/test_post_always.sh

@@ -39,11 +39,11 @@ esac
 
 mkdir -p "$test_log_dir"
 
-rsync -v -rlpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ .
+rsync -v -rlpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ --exclude="venv" .
 
 # Copy any log files.  Use rsync filters here to allow us to specify
 # all files we want to copy, as it's much more flexible than using
 # standard wildcards.
 rsync -v -dpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":/tmp/ \
-      --filter="include daos*.log" --filter="include test.cov" \
+      --filter="include daos*.log" --filter="include code_coverage/*" \
       --filter="exclude *" "$test_log_dir/"

requirements-code-coverage.txt

@@ -0,0 +1,3 @@
+gcovr
+junit-xml
+pyyaml