opensourcemirror
/
netdata
mirror of https://github.com/netdata/netdata.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Fix build for the AWS Kinesis exporting connector (#9823) 

Co-authored-by: James Mills <prologic@shortcircuit.net.au>
This commit is contained in:
Vladimir Kobal 2020-09-21 10:53:05 +03:00 committed by GitHub
parent e1fafcc4b1
commit a8c4f9f2a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 173 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/dockerfiles/Dockerfile.build_test vendored
View File

@ -7,6 +7,5 @@ ENV PRE=${PRE}
COPY . /netdata
RUN chmod +x /netdata/rmjsonc.sh
RUN /bin/sh /netdata/prep-cmd.sh
RUN /netdata/packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all

82
.github/workflows/build-and-install.yml vendored
View File

@ -108,7 +108,7 @@ jobs:
RMJSONC: ${{ matrix.rmjsonc }}
run: |
echo $PRE > ./prep-cmd.sh
echo $RMJSONC > ./rmjsonc.sh
echo $RMJSONC > ./rmjsonc.sh && chmod +x ./rmjsonc.sh
docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
- name: Regular build on ${{ matrix.distro }}
run: |
@ -124,3 +124,83 @@ jobs:
run: |
docker run -w /netdata test \
/bin/sh -c '/netdata/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
aws-kinesis-build:
name: With AWS Kinesis SDK
strategy:
fail-fast: false
matrix:
distro:
# XXX: CentOS 8.x is broken (libjudy)
# - 'centos:8'
- 'debian:buster'
- 'fedora:32'
- 'ubuntu:20.04'
include:
#- distro: 'centos:8'
# pre: >-
# yum -y update &&
# yum -y groupinstall 'Development Tools' &&
# yum -y install libcurl-devel openssl-devel libuuid-devel
# build_kinesis: >-
# git clone https://github.com/aws/aws-sdk-cpp.git &&
# cmake -DCMAKE_INSTALL_PREFIX=/usr
# -DBUILD_ONLY=kinesis
# ./aws-sdk-cpp &&
# make &&
# make install
- distro: 'debian:buster'
pre: >-
apt-get update &&
apt-get install -y build-essential &&
apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
build_kinesis: >-
git clone https://github.com/aws/aws-sdk-cpp.git &&
cmake -DCMAKE_INSTALL_PREFIX=/usr
-DBUILD_ONLY=kinesis
./aws-sdk-cpp &&
make &&
make install
- distro: 'fedora:32'
pre: >-
dnf -y update &&
dnf -y groupinstall 'Development Tools' &&
dnf -y install libcurl-devel openssl-devel libuuid-devel
build_kinesis: >-
git clone https://github.com/aws/aws-sdk-cpp.git &&
cmake -DCMAKE_INSTALL_PREFIX=/usr
-DBUILD_ONLY=kinesis
./aws-sdk-cpp &&
make &&
make install
- distro: 'ubuntu:20.04'
pre: >-
apt-get update &&
apt-get install -y build-essential &&
apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
build_kinesis: >-
git clone https://github.com/aws/aws-sdk-cpp.git &&
cmake -DCMAKE_INSTALL_PREFIX=/usr
-DBUILD_ONLY=kinesis
./aws-sdk-cpp &&
make &&
make install
runs-on: ubuntu-latest
steps:
- name: Git clone repository
uses: actions/checkout@v2
- name: install-required-packages.sh on ${{ matrix.distro }}
env:
PRE: ${{ matrix.pre }}
BUILD_KINESIS: ${{ matrix.build_kinesis }}
run: |
echo $PRE > ./prep-cmd.sh
echo $BUILD_KINESIS > ./build-kinesis.sh && chmod +x ./build-kinesis.sh
docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
- name: Build on ${{ matrix.distro }}
env:
RUNCMD: >-
./build-kinesis.sh &&
./netdata-installer.sh --dont-wait --dont-start-it --enable-backend-kinesis
run: |
docker run -w /netdata test \
/bin/sh -c "$RUNCMD"

43
CMakeLists.txt
View File

@ -237,27 +237,51 @@ pkg_check_modules(CURL libcurl)
# ${CURL_CFLAGS_OTHER}
# ${CURL_INCLUDE_DIRS}
# -----------------------------------------------------------------------------
# Detect libaws-c-common
find_library(HAVE_AWS_CHECKSUMS aws-checksums)
# later we use:
# ${HAVE_AWS_CHECKSUMS}
# -----------------------------------------------------------------------------
# Detect libaws-c-common
find_library(HAVE_AWS_COMMON aws-c-common)
# later we use:
# ${HAVE_AWS_COMMON}
# -----------------------------------------------------------------------------
# Detect libaws-c-common
find_library(HAVE_AWS_EVENT_STREAM aws-c-event-stream)
# later we use:
# ${HAVE_AWS_EVENT_STREAM}
# -----------------------------------------------------------------------------
# Detect libaws-cpp-sdk-core
find_library(HAVE_AWS aws-cpp-sdk-core)
pkg_check_modules(AWS_CORE aws-cpp-sdk-core)
# later we use:
# ${HAVE_AWS}
# ${AWS_CORE_LIBRARIES}
# ${AWS_CORE_CFLAGS_OTHER}
# ${AWS_CORE_INCLUDE_DIRS}
# -----------------------------------------------------------------------------
# Detect libaws-cpp-sdk-kinesis
find_library(HAVE_KINESIS aws-cpp-sdk-kinesis)
pkg_check_modules(KINESIS aws-cpp-sdk-kinesis)
# later we use:
# ${HAVE_KINESIS}
# ${KINESIS_LIBRARIES}
# ${KINESIS_CFLAGS_OTHER}
# ${KINESIS_INCLUDE_DIRS}
# -----------------------------------------------------------------------------
# Detect libgrpc
pkg_check_modules(GRPC grpc)
# later we use:
# ${GRPCF_LIBRARIES}
# ${GRPC_LIBRARIES}
# ${GRPC_CFLAGS_OTHER}
# ${GRPC_INCLUDE_DIRS}
@ -844,7 +868,8 @@ add_definitions(
# -----------------------------------------------------------------------------
# kinesis backend
IF(HAVE_KINESIS AND HAVE_AWS AND CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
IF(KINESIS_LIBRARIES AND AWS_CORE_LIBRARIES AND HAVE_AWS_EVENT_STREAM AND HAVE_AWS_COMMON AND HAVE_AWS_CHECKSUMS AND
CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
SET(ENABLE_BACKEND_KINESIS True)
ELSE()
SET(ENABLE_BACKEND_KINESIS False)
@ -853,8 +878,10 @@ ENDIF()
IF(ENABLE_BACKEND_KINESIS)
message(STATUS "kinesis backend: enabled")
list(APPEND NETDATA_FILES ${KINESIS_BACKEND_FILES} ${KINESIS_EXPORTING_FILES})
list(APPEND NETDATA_COMMON_LIBRARIES aws-cpp-sdk-kinesis aws-cpp-sdk-core ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
list(APPEND NETDATA_COMMON_LIBRARIES ${KINESIS_LIBRARIES} ${AWS_CORE_LIBRARIES}
${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${KINESIS_INCLUDE_DIRS} ${AWS_CORE_INCLUDE_DIRS}
${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
list(APPEND NETDATA_COMMON_CFLAGS ${CRYPTO_CFLAGS_OTHER} ${SSL_CFLAGS_OTHER} ${CURL_CFLAGS_OTHER})
ELSE()
message(STATUS "kinesis backend: disabled (requires AWS SDK for C++)")

87
configure.ac
View File

@ -1074,46 +1074,45 @@ PKG_CHECK_MODULES(
[have_libcurl=no]
)
AC_CHECK_LIB(
PKG_CHECK_MODULES(
[AWS_CPP_SDK_CORE],
[aws-cpp-sdk-core],
[cJSON_free],
[have_libaws_cpp_sdk_core=yes],
[have_libaws_cpp_sdk_core=no],
[${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}]
[AC_CHECK_LIB(
[aws-cpp-sdk-core],
[cJSON_free],
[have_libaws_cpp_sdk_core=yes],
[have_libaws_cpp_sdk_core=no]
)],
[have_libaws_cpp_sdk_core=no]
)
AC_MSG_CHECKING([for Aws::Kinesis::Model::PutRecordRequest in -laws-cpp-sdk-kinesis])
PKG_CHECK_MODULES(
[AWS_CPP_SDK_KINESIS],
[aws-cpp-sdk-kinesis],
[have_libaws_cpp_sdk_kinesis=yes],
[have_libaws_cpp_sdk_kinesis=no]
)
if test "${have_libaws_cpp_sdk_core}" = "yes" -a "${have_libcrypto}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcurl}" = "yes"; then
AC_LANG_SAVE
AC_LANG_CPLUSPLUS
save_LIBS="${LIBS}"
LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
save_CXXFLAGS="${CXXFLAGS}"
CXXFLAGS="${CXXFLAGS} -std=c++11"
AC_CHECK_LIB(
[aws-checksums],
[aws_checksums_crc32],
[have_libaws_checksums=yes],
[have_libaws_checksums=no]
)
AC_TRY_LINK(
[
#include <aws/core/Aws.h>
#include <aws/core/client/ClientConfiguration.h>
#include <aws/core/auth/AWSCredentials.h>
#include <aws/core/utils/Outcome.h>
#include <aws/kinesis/KinesisClient.h>
#include <aws/kinesis/model/PutRecordRequest.h>
],
[Aws::Kinesis::Model::PutRecordRequest request;],
[have_libaws_cpp_sdk_kinesis=yes],
[have_libaws_cpp_sdk_kinesis=no]
)
AC_CHECK_LIB(
[aws-c-common],
[aws_default_allocator],
[have_libaws_c_common=yes],
[have_libaws_c_common=no]
)
LIBS="${save_LIBS}"
CXXFLAGS="${save_CXXFLAGS}"
AC_LANG_RESTORE
else
have_libaws_cpp_sdk_kinesis=no
fi
AC_MSG_RESULT([${have_libaws_cpp_sdk_kinesis}])
AC_CHECK_LIB(
[aws-c-event-stream],
[aws_event_stream_library_init],
[have_libaws_c_event_stream=yes],
[have_libaws_c_event_stream=no]
)
test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \
AC_MSG_ERROR([libaws-cpp-sdk-kinesis required but not found. try installing AWS C++ SDK])
@ -1130,14 +1129,26 @@ test "${enable_backend_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \
test "${enable_backend_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \
AC_MSG_ERROR([libcrypto required but not found])
test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_checksums}" != "yes" \
-a "${have_libaws_c_common}" != "yes" \
-a "${have_libaws_c_event_stream}" != "yes" && \
AC_MSG_ERROR([AWS SKD third party dependencies required but not found])
AC_MSG_CHECKING([if kinesis backend should be enabled])
if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" = "yes" \
-a "${have_libcurl}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcrypto}" = "yes"; then
if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" \
-a "${have_libaws_cpp_sdk_core}" = "yes" \
-a "${have_libaws_checksums}" = "yes" \
-a "${have_libaws_c_common}" = "yes" \
-a "${have_libaws_c_event_stream}" = "yes" \
-a "${have_libcurl}" = "yes" \
-a "${have_libssl}" = "yes" \
-a "${have_libcrypto}" = "yes"; then
enable_backend_kinesis="yes"
AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability])
OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}"
CXX11FLAG="-std=c++11"
OPTIONAL_KINESIS_LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
CXX11FLAG="${AWS_CPP_SDK_KINESIS_CFLAGS} ${AWS_CPP_SDK_CORE_CFLAGS}"
OPTIONAL_KINESIS_LIBS="${AWS_CPP_SDK_KINESIS_LIBS} ${AWS_CPP_SDK_CORE_LIBS} \
${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
else
enable_backend_kinesis="no"
fi

13
exporting/aws_kinesis/README.md
View File

@ -10,20 +10,23 @@ sidebar_label: AWS Kinesis Data Streams
## Prerequisites
To use AWS Kinesis for metric collecting and processing, you should first
[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++. Netdata
works with the SDK version 1.7.121. Other versions might work correctly as well, but they were not tested with Netdata.
[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++.
`libcrypto`, `libssl`, and `libcurl` are also required to compile Netdata with Kinesis support enabled. Next, Netdata
should be re-installed from the source. The installer will detect that the required libraries are now available.
If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY="kinesis"`. Otherwise, the
building process could take a very long time. Note that the default installation path for the libraries is
If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY=kinesis`. Otherwise, the
build process could take a very long time. Note, that the default installation path for the libraries is
`/usr/local/lib64`. Many Linux distributions don't include this path as the default one for a library search, so it is
advisable to use the following options to `cmake` while building the AWS SDK:
```sh
cmake -DCMAKE_INSTALL_LIBDIR=/usr/lib -DCMAKE_INSTALL_INCLUDEDIR=/usr/include -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
sudo cmake -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
```
The `-DCMAKE_INSTALL_PREFIX=/usr` option also ensures that
[third party dependencies](https://github.com/aws/aws-sdk-cpp#third-party-dependencies) are installed in your system
during the SDK build process.
## Configuration
To enable data sending to the Kinesis service, run `./edit-config exporting.conf` in the Netdata configuration directory