diff --git a/.github/dockerfiles/Dockerfile.build_test b/.github/dockerfiles/Dockerfile.build_test index 5e6de6d603..1dc3e303d6 100644 --- a/.github/dockerfiles/Dockerfile.build_test +++ b/.github/dockerfiles/Dockerfile.build_test @@ -7,6 +7,5 @@ ENV PRE=${PRE} COPY . /netdata -RUN chmod +x /netdata/rmjsonc.sh RUN /bin/sh /netdata/prep-cmd.sh RUN /netdata/packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml index 40eb269cdf..cf8b212e03 100644 --- a/.github/workflows/build-and-install.yml +++ b/.github/workflows/build-and-install.yml @@ -108,7 +108,7 @@ jobs: RMJSONC: ${{ matrix.rmjsonc }} run: | echo $PRE > ./prep-cmd.sh - echo $RMJSONC > ./rmjsonc.sh + echo $RMJSONC > ./rmjsonc.sh && chmod +x ./rmjsonc.sh docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} - name: Regular build on ${{ matrix.distro }} run: | @@ -124,3 +124,83 @@ jobs: run: | docker run -w /netdata test \ /bin/sh -c '/netdata/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud' + aws-kinesis-build: + name: With AWS Kinesis SDK + strategy: + fail-fast: false + matrix: + distro: + # XXX: CentOS 8.x is broken (libjudy) + # - 'centos:8' + - 'debian:buster' + - 'fedora:32' + - 'ubuntu:20.04' + include: + #- distro: 'centos:8' + # pre: >- + # yum -y update && + # yum -y groupinstall 'Development Tools' && + # yum -y install libcurl-devel openssl-devel libuuid-devel + # build_kinesis: >- + # git clone https://github.com/aws/aws-sdk-cpp.git && + # cmake -DCMAKE_INSTALL_PREFIX=/usr + # -DBUILD_ONLY=kinesis + # ./aws-sdk-cpp && + # make && + # make install + - distro: 'debian:buster' + pre: >- + apt-get update && + apt-get install -y build-essential && + apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev + build_kinesis: >- + git clone https://github.com/aws/aws-sdk-cpp.git && + cmake -DCMAKE_INSTALL_PREFIX=/usr + -DBUILD_ONLY=kinesis + ./aws-sdk-cpp && + make && + make install + - distro: 'fedora:32' + pre: >- + dnf -y update && + dnf -y groupinstall 'Development Tools' && + dnf -y install libcurl-devel openssl-devel libuuid-devel + build_kinesis: >- + git clone https://github.com/aws/aws-sdk-cpp.git && + cmake -DCMAKE_INSTALL_PREFIX=/usr + -DBUILD_ONLY=kinesis + ./aws-sdk-cpp && + make && + make install + - distro: 'ubuntu:20.04' + pre: >- + apt-get update && + apt-get install -y build-essential && + apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev + build_kinesis: >- + git clone https://github.com/aws/aws-sdk-cpp.git && + cmake -DCMAKE_INSTALL_PREFIX=/usr + -DBUILD_ONLY=kinesis + ./aws-sdk-cpp && + make && + make install + runs-on: ubuntu-latest + steps: + - name: Git clone repository + uses: actions/checkout@v2 + - name: install-required-packages.sh on ${{ matrix.distro }} + env: + PRE: ${{ matrix.pre }} + BUILD_KINESIS: ${{ matrix.build_kinesis }} + run: | + echo $PRE > ./prep-cmd.sh + echo $BUILD_KINESIS > ./build-kinesis.sh && chmod +x ./build-kinesis.sh + docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} + - name: Build on ${{ matrix.distro }} + env: + RUNCMD: >- + ./build-kinesis.sh && + ./netdata-installer.sh --dont-wait --dont-start-it --enable-backend-kinesis + run: | + docker run -w /netdata test \ + /bin/sh -c "$RUNCMD" diff --git a/CMakeLists.txt b/CMakeLists.txt index 4633b7bc9c..3278411a79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,27 +237,51 @@ pkg_check_modules(CURL libcurl) # ${CURL_CFLAGS_OTHER} # ${CURL_INCLUDE_DIRS} +# ----------------------------------------------------------------------------- +# Detect libaws-c-common + +find_library(HAVE_AWS_CHECKSUMS aws-checksums) +# later we use: +# ${HAVE_AWS_CHECKSUMS} + +# ----------------------------------------------------------------------------- +# Detect libaws-c-common + +find_library(HAVE_AWS_COMMON aws-c-common) +# later we use: +# ${HAVE_AWS_COMMON} + +# ----------------------------------------------------------------------------- +# Detect libaws-c-common + +find_library(HAVE_AWS_EVENT_STREAM aws-c-event-stream) +# later we use: +# ${HAVE_AWS_EVENT_STREAM} # ----------------------------------------------------------------------------- # Detect libaws-cpp-sdk-core -find_library(HAVE_AWS aws-cpp-sdk-core) +pkg_check_modules(AWS_CORE aws-cpp-sdk-core) # later we use: -# ${HAVE_AWS} +# ${AWS_CORE_LIBRARIES} +# ${AWS_CORE_CFLAGS_OTHER} +# ${AWS_CORE_INCLUDE_DIRS} # ----------------------------------------------------------------------------- # Detect libaws-cpp-sdk-kinesis -find_library(HAVE_KINESIS aws-cpp-sdk-kinesis) +pkg_check_modules(KINESIS aws-cpp-sdk-kinesis) # later we use: -# ${HAVE_KINESIS} +# ${KINESIS_LIBRARIES} +# ${KINESIS_CFLAGS_OTHER} +# ${KINESIS_INCLUDE_DIRS} # ----------------------------------------------------------------------------- # Detect libgrpc pkg_check_modules(GRPC grpc) # later we use: -# ${GRPCF_LIBRARIES} +# ${GRPC_LIBRARIES} # ${GRPC_CFLAGS_OTHER} # ${GRPC_INCLUDE_DIRS} @@ -844,7 +868,8 @@ add_definitions( # ----------------------------------------------------------------------------- # kinesis backend -IF(HAVE_KINESIS AND HAVE_AWS AND CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES) +IF(KINESIS_LIBRARIES AND AWS_CORE_LIBRARIES AND HAVE_AWS_EVENT_STREAM AND HAVE_AWS_COMMON AND HAVE_AWS_CHECKSUMS AND + CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES) SET(ENABLE_BACKEND_KINESIS True) ELSE() SET(ENABLE_BACKEND_KINESIS False) @@ -853,8 +878,10 @@ ENDIF() IF(ENABLE_BACKEND_KINESIS) message(STATUS "kinesis backend: enabled") list(APPEND NETDATA_FILES ${KINESIS_BACKEND_FILES} ${KINESIS_EXPORTING_FILES}) - list(APPEND NETDATA_COMMON_LIBRARIES aws-cpp-sdk-kinesis aws-cpp-sdk-core ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES}) - list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS}) + list(APPEND NETDATA_COMMON_LIBRARIES ${KINESIS_LIBRARIES} ${AWS_CORE_LIBRARIES} + ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES}) + list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${KINESIS_INCLUDE_DIRS} ${AWS_CORE_INCLUDE_DIRS} + ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS}) list(APPEND NETDATA_COMMON_CFLAGS ${CRYPTO_CFLAGS_OTHER} ${SSL_CFLAGS_OTHER} ${CURL_CFLAGS_OTHER}) ELSE() message(STATUS "kinesis backend: disabled (requires AWS SDK for C++)") diff --git a/configure.ac b/configure.ac index f9379a9bfa..7d9938f8b3 100644 --- a/configure.ac +++ b/configure.ac @@ -1074,46 +1074,45 @@ PKG_CHECK_MODULES( [have_libcurl=no] ) -AC_CHECK_LIB( +PKG_CHECK_MODULES( + [AWS_CPP_SDK_CORE], [aws-cpp-sdk-core], - [cJSON_free], - [have_libaws_cpp_sdk_core=yes], - [have_libaws_cpp_sdk_core=no], - [${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}] + [AC_CHECK_LIB( + [aws-cpp-sdk-core], + [cJSON_free], + [have_libaws_cpp_sdk_core=yes], + [have_libaws_cpp_sdk_core=no] + )], + [have_libaws_cpp_sdk_core=no] ) -AC_MSG_CHECKING([for Aws::Kinesis::Model::PutRecordRequest in -laws-cpp-sdk-kinesis]) +PKG_CHECK_MODULES( + [AWS_CPP_SDK_KINESIS], + [aws-cpp-sdk-kinesis], + [have_libaws_cpp_sdk_kinesis=yes], + [have_libaws_cpp_sdk_kinesis=no] +) -if test "${have_libaws_cpp_sdk_core}" = "yes" -a "${have_libcrypto}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcurl}" = "yes"; then - AC_LANG_SAVE - AC_LANG_CPLUSPLUS - save_LIBS="${LIBS}" - LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}" - save_CXXFLAGS="${CXXFLAGS}" - CXXFLAGS="${CXXFLAGS} -std=c++11" +AC_CHECK_LIB( + [aws-checksums], + [aws_checksums_crc32], + [have_libaws_checksums=yes], + [have_libaws_checksums=no] +) - AC_TRY_LINK( - [ - #include - #include - #include - #include - #include - #include - ], - [Aws::Kinesis::Model::PutRecordRequest request;], - [have_libaws_cpp_sdk_kinesis=yes], - [have_libaws_cpp_sdk_kinesis=no] - ) +AC_CHECK_LIB( + [aws-c-common], + [aws_default_allocator], + [have_libaws_c_common=yes], + [have_libaws_c_common=no] +) - LIBS="${save_LIBS}" - CXXFLAGS="${save_CXXFLAGS}" - AC_LANG_RESTORE -else - have_libaws_cpp_sdk_kinesis=no -fi - -AC_MSG_RESULT([${have_libaws_cpp_sdk_kinesis}]) +AC_CHECK_LIB( + [aws-c-event-stream], + [aws_event_stream_library_init], + [have_libaws_c_event_stream=yes], + [have_libaws_c_event_stream=no] +) test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \ AC_MSG_ERROR([libaws-cpp-sdk-kinesis required but not found. try installing AWS C++ SDK]) @@ -1130,14 +1129,26 @@ test "${enable_backend_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \ test "${enable_backend_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \ AC_MSG_ERROR([libcrypto required but not found]) +test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_checksums}" != "yes" \ + -a "${have_libaws_c_common}" != "yes" \ + -a "${have_libaws_c_event_stream}" != "yes" && \ + AC_MSG_ERROR([AWS SKD third party dependencies required but not found]) + AC_MSG_CHECKING([if kinesis backend should be enabled]) -if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" = "yes" \ - -a "${have_libcurl}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcrypto}" = "yes"; then +if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" \ + -a "${have_libaws_cpp_sdk_core}" = "yes" \ + -a "${have_libaws_checksums}" = "yes" \ + -a "${have_libaws_c_common}" = "yes" \ + -a "${have_libaws_c_event_stream}" = "yes" \ + -a "${have_libcurl}" = "yes" \ + -a "${have_libssl}" = "yes" \ + -a "${have_libcrypto}" = "yes"; then enable_backend_kinesis="yes" AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability]) OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}" - CXX11FLAG="-std=c++11" - OPTIONAL_KINESIS_LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}" + CXX11FLAG="${AWS_CPP_SDK_KINESIS_CFLAGS} ${AWS_CPP_SDK_CORE_CFLAGS}" + OPTIONAL_KINESIS_LIBS="${AWS_CPP_SDK_KINESIS_LIBS} ${AWS_CPP_SDK_CORE_LIBS} \ + ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}" else enable_backend_kinesis="no" fi diff --git a/exporting/aws_kinesis/README.md b/exporting/aws_kinesis/README.md index 538ff34266..299fec5817 100644 --- a/exporting/aws_kinesis/README.md +++ b/exporting/aws_kinesis/README.md @@ -10,20 +10,23 @@ sidebar_label: AWS Kinesis Data Streams ## Prerequisites To use AWS Kinesis for metric collecting and processing, you should first -[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++. Netdata -works with the SDK version 1.7.121. Other versions might work correctly as well, but they were not tested with Netdata. +[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++. `libcrypto`, `libssl`, and `libcurl` are also required to compile Netdata with Kinesis support enabled. Next, Netdata should be re-installed from the source. The installer will detect that the required libraries are now available. -If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY="kinesis"`. Otherwise, the -building process could take a very long time. Note that the default installation path for the libraries is +If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY=kinesis`. Otherwise, the +build process could take a very long time. Note, that the default installation path for the libraries is `/usr/local/lib64`. Many Linux distributions don't include this path as the default one for a library search, so it is advisable to use the following options to `cmake` while building the AWS SDK: ```sh -cmake -DCMAKE_INSTALL_LIBDIR=/usr/lib -DCMAKE_INSTALL_INCLUDEDIR=/usr/include -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY=kinesis +sudo cmake -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_ONLY=kinesis ``` +The `-DCMAKE_INSTALL_PREFIX=/usr` option also ensures that +[third party dependencies](https://github.com/aws/aws-sdk-cpp#third-party-dependencies) are installed in your system +during the SDK build process. + ## Configuration To enable data sending to the Kinesis service, run `./edit-config exporting.conf` in the Netdata configuration directory