From 5657086b1cc70cace76467b025033091ac8c3e6b Mon Sep 17 00:00:00 2001 From: "Paul Emm. Katsoulakis" <34388743+paulkatsoulakis@users.noreply.github.com> Date: Thu, 12 Sep 2019 12:49:21 +0300 Subject: [PATCH] netdata/ci: second batch of fixes for coverity scan script and others (#6804) * netdata/ci: fine grain coverity scan toolkit 1) Deprecate coverity-install.sh 2) add set -e to raise errors more aggressively when something is wrong 3) refactor some variable definitions for temp and install paths, use mktemp and clean up temp dir when all ok * netdata/ci: reduce the scope * netdata/ci: require curl * netdata/ci: go soft on functions loading * netdata/ci: allow override of build version without touching the script * netdata/ci: handle shellcheck errors, some were silenced because we dont handle them * netdata/ci: coverity scan, parse any extra options other than --with-install and pass them to configure 1) add default configure arguments if no extras are given 2) parse all possible arguments individually, handle --with-install and pass the rest to scanit * netdata/ci: focus, argument parsing in one place a bit more consistently * netdata/ci: prepare for using * netdata/ci: coverity - add more flags, adjust deps list 1) Add with math and with zlib flags on the script 2) add xenstat dependencies 3) run in coverity with custom flags, so that we add xenstat (no need to be by default for the users of the script) * netdata/packaging: add xen-utils also * test the flow * netdata/ci: dont forget to go back to the original directory * netdata/ci: please, travis, stop it * netdata/ci: how about this then * netdata/ci: silence that, we cant call it otherwise * netdata/packaging: update documentation around xenstat on centos * netdata/packaging: dont enable xenstat at this stage * netdata/ci: add += instead of variable reusage --- .travis.yml | 5 +- Makefile.am | 1 - collectors/xenstat.plugin/README.md | 1 + coverity-install.sh | 8 -- coverity-scan.sh | 117 ++++++++++++++++++---------- packaging/DISTRIBUTIONS.md | 3 +- 6 files changed, 84 insertions(+), 51 deletions(-) delete mode 100755 coverity-install.sh diff --git a/.travis.yml b/.travis.yml index b5b258e95e..8d9cea2cad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,8 @@ stages: # Nightly operations - name: Nightly operations - if: branch = master AND type = cron + #if: branch = master AND type = cron + if: branch =fix-coverity-toolkit - name: Nightly release if: branch = master AND type = cron @@ -536,7 +537,7 @@ jobs: - post_message "TRAVIS_MESSAGE" "Starting nightly operations" "${NOTIF_CHANNEL}" - bash <(curl -sS https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh) --dont-wait --non-interactive netdata - sudo apt-get install -y libjson-c-dev libipmimonitoring-dev libcups2-dev libsnappy-dev libprotobuf-dev libprotoc-dev libssl-dev protobuf-compiler - script: ./coverity-install.sh && ./coverity-scan.sh + script: ./coverity-scan.sh --with-install after_failure: post_message "TRAVIS_MESSAGE" " Coverity nightly run has failed" "${NOTIF_CHANNEL}" env: - ALLOW_SOFT_FAILURE_HERE=true diff --git a/Makefile.am b/Makefile.am index fa12a757d2..0e515db8d3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -72,7 +72,6 @@ dist_noinst_DATA= \ # should be proper init.d/openrc/systemd usable dist_noinst_SCRIPTS= \ coverity-scan.sh \ - coverity-install.sh \ packaging/installer/netdata-updater.sh \ packaging/installer/netdata-uninstaller.sh \ packaging/installer/kickstart.sh \ diff --git a/collectors/xenstat.plugin/README.md b/collectors/xenstat.plugin/README.md index 9943ba1af3..4fa047e31d 100644 --- a/collectors/xenstat.plugin/README.md +++ b/collectors/xenstat.plugin/README.md @@ -5,6 +5,7 @@ ## Prerequisites 1. install `xen-dom0-libs-devel` and `yajl-devel` using the package manager of your system. + Note: On Cent-OS systems you will need `centos-release-xen` repository and the required package for xen is `xen-devel` 2. re-install Netdata from source. The installer will detect that the required libraries are now available and will also build xenstat.plugin. diff --git a/coverity-install.sh b/coverity-install.sh deleted file mode 100755 index 83d2ad92f5..0000000000 --- a/coverity-install.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# Coverity installation script -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author: Pavlos Emm. Katsoulakis (paul@netdata.cloud) - -exec ./coverity-scan.sh install "${@}" diff --git a/coverity-scan.sh b/coverity-scan.sh index ee8f19e7fc..dd58b47884 100755 --- a/coverity-scan.sh +++ b/coverity-scan.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Coverity scan script # # Copyright: SPDX-License-Identifier: GPL-3.0-or-later @@ -6,6 +7,7 @@ # Author : Costa Tsaousis (costa@netdata.cloud) # Author : Pawel Krupa (paulfantom) # Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +# shellcheck disable=SC1091,SC2230,SC2086 # To run manually, save configuration to .coverity-scan.conf like this: # @@ -25,22 +27,30 @@ # this includes the token, so the default is not to print it. # COVERITY_SUBMIT_DEBUG=1 # +# Override the standard coverity build version we know is supported +# COVERITY_BUILD_VERSION="cov-analysis-linux64-2019.03" +# # All these variables can also be exported before running this script. # # If the first parameter of this script is "install", # coverity build tools will be downloaded and installed in /opt/coverity -# the version of coverity to use -COVERITY_BUILD_VERSION="cov-analysis-linux64-2019.03" +set -e -source packaging/installer/functions.sh || exit 1 +INSTALL_DIR="/opt" + +# the version of coverity to use +COVERITY_BUILD_VERSION="${COVERITY_BUILD_VERSION:-cov-analysis-linux64-2019.03}" + +# TODO: For some reasons this does not fully load on Debian 10 (Haven't checked if it happens on other distros yet), it breaks +source packaging/installer/functions.sh || echo "Failed to fully load the functions library" cpus=$(find_processors) [ -z "${cpus}" ] && cpus=1 if [ -f ".coverity-scan.conf" ] then - source ".coverity-scan.conf" || exit 1 + source ".coverity-scan.conf" fi repo="${REPOSITORY}" @@ -59,6 +69,10 @@ if [ -z "${token}" ]; then fatal "export variable COVERITY_SCAN_TOKEN or set it in .coverity-scan.conf" fi +if ! command -v curl >/dev/null 2>&1; then + fatal "CURL is required for coverity scan to work" +fi + # only print the output of a command # when debugging is enabled # used to hide the token when debugging is not enabled @@ -74,9 +88,11 @@ debugrun() { } scanit() { - export PATH="${PATH}:/opt/${COVERITY_BUILD_VERSION}/bin/" + progress "Scanning using coverity" + export PATH="${PATH}:${INSTALL_DIR}/${COVERITY_BUILD_VERSION}/bin/" covbuild="${COVERITY_BUILD_PATH}" [ -z "${covbuild}" ] && covbuild="$(which cov-build 2>/dev/null || command -v cov-build 2>/dev/null)" + if [ -z "${covbuild}" ]; then fatal "Cannot find 'cov-build' binary in \$PATH. Export variable COVERITY_BUILD_PATH or set it in .coverity-scan.conf" elif [ ! -x "${covbuild}" ]; then @@ -94,54 +110,45 @@ scanit() { [ -f netdata-coverity-analysis.tgz ] && run rm netdata-coverity-analysis.tgz progress "Configuring netdata source..." - run autoreconf -ivf - run ./configure --disable-lto \ - --enable-https \ - --enable-jsonc \ - --enable-plugin-nfacct \ - --enable-plugin-freeipmi \ - --enable-plugin-cups \ - --enable-backend-prometheus-remote-write \ - ${NULL} - # TODO: enable these plugins too - # --enable-plugin-xenstat \ - # --enable-backend-kinesis \ - # --enable-backend-mongodb \ + run autoreconf -ivf + run ./configure ${OTHER_OPTIONS} progress "Analyzing netdata..." - run "${covbuild}" --dir cov-int make -j${cpus} || exit 1 + run "${covbuild}" --dir cov-int make -j${cpus} echo >&2 "Compressing analysis..." - run tar czvf netdata-coverity-analysis.tgz cov-int || exit 1 + run tar czvf netdata-coverity-analysis.tgz cov-int echo >&2 "Sending analysis to coverity for netdata version ${version} ..." COVERITY_SUBMIT_RESULT=$(debugrun curl --progress-bar \ --form token="${token}" \ - --form email=${email} \ + --form email="${email}" \ --form file=@netdata-coverity-analysis.tgz \ --form version="${version}" \ --form description="netdata, monitor everything, in real-time." \ - https://scan.coverity.com/builds?project=${repo}) + https://scan.coverity.com/builds?project="${repo}") - echo ${COVERITY_SUBMIT_RESULT} | grep -q -e 'Build successfully submitted' || echo >&2 "scan results were not pushed to coverity. Message was: ${COVERITY_SUBMIT_RESULT}" + echo "${COVERITY_SUBMIT_RESULT}" | grep -q -e 'Build successfully submitted' || echo >&2 "scan results were not pushed to coverity. Message was: ${COVERITY_SUBMIT_RESULT}" progress "Coverity scan completed" } installit() { - progress "Downloading coverity..." - cd /tmp || exit 1 + ORIGINAL_DIR="${PWD}" + TMP_DIR="$(mktemp -d /tmp/netdata-coverity-scan-XXXXX)" + progress "Downloading coverity in ${TMP_DIR}..." + cd "${TMP_DIR}" - [ -f "${COVERITY_BUILD_VERSION}.tar.gz" ] && run rm -f "${COVERITY_BUILD_VERSION}.tar.gz" debugrun curl --remote-name --remote-header-name --show-error --location --data "token=${token}&project=${repo}" https://scan.coverity.com/download/linux64 if [ -f "${COVERITY_BUILD_VERSION}.tar.gz" ]; then progress "Installing coverity..." - cd /opt || exit 1 - run sudo tar -z -x -f "/tmp/${COVERITY_BUILD_VERSION}.tar.gz" || exit 1 - rm "/tmp/${COVERITY_BUILD_VERSION}.tar.gz" - export PATH=${PATH}:/opt/${COVERITY_BUILD_VERSION}/bin/ + cd "${INSTALL_DIR}" + + run sudo tar -z -x -f "${TMP_DIR}/${COVERITY_BUILD_VERSION}.tar.gz" || exit 1 + rm "${TMP_DIR}/${COVERITY_BUILD_VERSION}.tar.gz" + export PATH=${PATH}:${INSTALL_DIR}/${COVERITY_BUILD_VERSION}/bin/ else fatal "Failed to download coverity tool tarball!" fi @@ -152,16 +159,48 @@ installit() { fatal "Failed to install coverity." fi + # Clean temp directory + [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" + progress "Coverity scan tools are installed." + cd "$ORIGINAL_DIR" return 0 } -if [ "${1}" = "install" ] -then - shift 1 - installit "${@}" - exit $? -else - scanit "${@}" - exit $? -fi +OTHER_OPTIONS="--disable-lto" +OTHER_OPTIONS+=" --with-zlib" +OTHER_OPTIONS+=" --with-math" +OTHER_OPTIONS+=" --enable-https" +OTHER_OPTIONS+=" --enable-jsonc" +OTHER_OPTIONS+=" --enable-plugin-nfacct" +OTHER_OPTIONS+=" --enable-plugin-freeipmi" +OTHER_OPTIONS+=" --enable-plugin-cups" +OTHER_OPTIONS+=" --enable-backend-prometheus-remote-write" +# TODO: enable these plugins too +#OTHER_OPTIONS+=" --enable-plugin-xenstat" +#OTHER_OPTIONS+=" --enable-backend-kinesis" +#OTHER_OPTIONS+=" --enable-backend-mongodb" + +FOUND_OPTS="NO" +while [ -n "${1}" ]; do + if [ "${1}" = "--with-install" ]; then + progress "Running coverity install" + installit + shift 1 + elif [ -n "${1}" ]; then + # Clear the default arguments, once you bump into the first argument + if [ "${FOUND_OPTS}" = "NO" ]; then + OTHER_OPTIONS="${1}" + FOUND_OPTS="YES" + else + OTHER_OPTIONS+=" ${1}" + fi + + shift 1 + else + break + fi +done + +echo "Running coverity scan with extra options ${OTHER_OPTIONS}" +scanit "${OTHER_OPTIONS}" diff --git a/packaging/DISTRIBUTIONS.md b/packaging/DISTRIBUTIONS.md index a8625e6ff1..286e2dd10a 100644 --- a/packaging/DISTRIBUTIONS.md +++ b/packaging/DISTRIBUTIONS.md @@ -188,7 +188,8 @@ This is Netdata's TLS capability that incorporates encryption on the web server - **Flags/instructions to enable**: None - **Flags to disable from source**: --disable-plugin-xenstat -- **What packages required for auto-detect?**: `xen-dom0-libs-devel`, `yajl-dev` +- **What packages required for auto-detect?**: `xen-dom0-libs-devel or xen-devel`, `yajl-dev or yajl-devel` + Note: for cent-OS based systems you will need `centos-release-xen` repository to get xen-devel #### CUPS