Fix build for the AWS Kinesis exporting connector (#9823)
Co-authored-by: James Mills <prologic@shortcircuit.net.au>
This commit is contained in:
parent
e1fafcc4b1
commit
a8c4f9f2a9
|
@ -7,6 +7,5 @@ ENV PRE=${PRE}
|
|||
|
||||
COPY . /netdata
|
||||
|
||||
RUN chmod +x /netdata/rmjsonc.sh
|
||||
RUN /bin/sh /netdata/prep-cmd.sh
|
||||
RUN /netdata/packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
|
||||
|
|
|
@ -108,7 +108,7 @@ jobs:
|
|||
RMJSONC: ${{ matrix.rmjsonc }}
|
||||
run: |
|
||||
echo $PRE > ./prep-cmd.sh
|
||||
echo $RMJSONC > ./rmjsonc.sh
|
||||
echo $RMJSONC > ./rmjsonc.sh && chmod +x ./rmjsonc.sh
|
||||
docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
|
||||
- name: Regular build on ${{ matrix.distro }}
|
||||
run: |
|
||||
|
@ -124,3 +124,83 @@ jobs:
|
|||
run: |
|
||||
docker run -w /netdata test \
|
||||
/bin/sh -c '/netdata/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
|
||||
aws-kinesis-build:
|
||||
name: With AWS Kinesis SDK
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
distro:
|
||||
# XXX: CentOS 8.x is broken (libjudy)
|
||||
# - 'centos:8'
|
||||
- 'debian:buster'
|
||||
- 'fedora:32'
|
||||
- 'ubuntu:20.04'
|
||||
include:
|
||||
#- distro: 'centos:8'
|
||||
# pre: >-
|
||||
# yum -y update &&
|
||||
# yum -y groupinstall 'Development Tools' &&
|
||||
# yum -y install libcurl-devel openssl-devel libuuid-devel
|
||||
# build_kinesis: >-
|
||||
# git clone https://github.com/aws/aws-sdk-cpp.git &&
|
||||
# cmake -DCMAKE_INSTALL_PREFIX=/usr
|
||||
# -DBUILD_ONLY=kinesis
|
||||
# ./aws-sdk-cpp &&
|
||||
# make &&
|
||||
# make install
|
||||
- distro: 'debian:buster'
|
||||
pre: >-
|
||||
apt-get update &&
|
||||
apt-get install -y build-essential &&
|
||||
apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
|
||||
build_kinesis: >-
|
||||
git clone https://github.com/aws/aws-sdk-cpp.git &&
|
||||
cmake -DCMAKE_INSTALL_PREFIX=/usr
|
||||
-DBUILD_ONLY=kinesis
|
||||
./aws-sdk-cpp &&
|
||||
make &&
|
||||
make install
|
||||
- distro: 'fedora:32'
|
||||
pre: >-
|
||||
dnf -y update &&
|
||||
dnf -y groupinstall 'Development Tools' &&
|
||||
dnf -y install libcurl-devel openssl-devel libuuid-devel
|
||||
build_kinesis: >-
|
||||
git clone https://github.com/aws/aws-sdk-cpp.git &&
|
||||
cmake -DCMAKE_INSTALL_PREFIX=/usr
|
||||
-DBUILD_ONLY=kinesis
|
||||
./aws-sdk-cpp &&
|
||||
make &&
|
||||
make install
|
||||
- distro: 'ubuntu:20.04'
|
||||
pre: >-
|
||||
apt-get update &&
|
||||
apt-get install -y build-essential &&
|
||||
apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
|
||||
build_kinesis: >-
|
||||
git clone https://github.com/aws/aws-sdk-cpp.git &&
|
||||
cmake -DCMAKE_INSTALL_PREFIX=/usr
|
||||
-DBUILD_ONLY=kinesis
|
||||
./aws-sdk-cpp &&
|
||||
make &&
|
||||
make install
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Git clone repository
|
||||
uses: actions/checkout@v2
|
||||
- name: install-required-packages.sh on ${{ matrix.distro }}
|
||||
env:
|
||||
PRE: ${{ matrix.pre }}
|
||||
BUILD_KINESIS: ${{ matrix.build_kinesis }}
|
||||
run: |
|
||||
echo $PRE > ./prep-cmd.sh
|
||||
echo $BUILD_KINESIS > ./build-kinesis.sh && chmod +x ./build-kinesis.sh
|
||||
docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
|
||||
- name: Build on ${{ matrix.distro }}
|
||||
env:
|
||||
RUNCMD: >-
|
||||
./build-kinesis.sh &&
|
||||
./netdata-installer.sh --dont-wait --dont-start-it --enable-backend-kinesis
|
||||
run: |
|
||||
docker run -w /netdata test \
|
||||
/bin/sh -c "$RUNCMD"
|
||||
|
|
|
@ -237,27 +237,51 @@ pkg_check_modules(CURL libcurl)
|
|||
# ${CURL_CFLAGS_OTHER}
|
||||
# ${CURL_INCLUDE_DIRS}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Detect libaws-c-common
|
||||
|
||||
find_library(HAVE_AWS_CHECKSUMS aws-checksums)
|
||||
# later we use:
|
||||
# ${HAVE_AWS_CHECKSUMS}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Detect libaws-c-common
|
||||
|
||||
find_library(HAVE_AWS_COMMON aws-c-common)
|
||||
# later we use:
|
||||
# ${HAVE_AWS_COMMON}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Detect libaws-c-common
|
||||
|
||||
find_library(HAVE_AWS_EVENT_STREAM aws-c-event-stream)
|
||||
# later we use:
|
||||
# ${HAVE_AWS_EVENT_STREAM}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Detect libaws-cpp-sdk-core
|
||||
|
||||
find_library(HAVE_AWS aws-cpp-sdk-core)
|
||||
pkg_check_modules(AWS_CORE aws-cpp-sdk-core)
|
||||
# later we use:
|
||||
# ${HAVE_AWS}
|
||||
# ${AWS_CORE_LIBRARIES}
|
||||
# ${AWS_CORE_CFLAGS_OTHER}
|
||||
# ${AWS_CORE_INCLUDE_DIRS}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Detect libaws-cpp-sdk-kinesis
|
||||
|
||||
find_library(HAVE_KINESIS aws-cpp-sdk-kinesis)
|
||||
pkg_check_modules(KINESIS aws-cpp-sdk-kinesis)
|
||||
# later we use:
|
||||
# ${HAVE_KINESIS}
|
||||
# ${KINESIS_LIBRARIES}
|
||||
# ${KINESIS_CFLAGS_OTHER}
|
||||
# ${KINESIS_INCLUDE_DIRS}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Detect libgrpc
|
||||
|
||||
pkg_check_modules(GRPC grpc)
|
||||
# later we use:
|
||||
# ${GRPCF_LIBRARIES}
|
||||
# ${GRPC_LIBRARIES}
|
||||
# ${GRPC_CFLAGS_OTHER}
|
||||
# ${GRPC_INCLUDE_DIRS}
|
||||
|
||||
|
@ -844,7 +868,8 @@ add_definitions(
|
|||
# -----------------------------------------------------------------------------
|
||||
# kinesis backend
|
||||
|
||||
IF(HAVE_KINESIS AND HAVE_AWS AND CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
|
||||
IF(KINESIS_LIBRARIES AND AWS_CORE_LIBRARIES AND HAVE_AWS_EVENT_STREAM AND HAVE_AWS_COMMON AND HAVE_AWS_CHECKSUMS AND
|
||||
CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
|
||||
SET(ENABLE_BACKEND_KINESIS True)
|
||||
ELSE()
|
||||
SET(ENABLE_BACKEND_KINESIS False)
|
||||
|
@ -853,8 +878,10 @@ ENDIF()
|
|||
IF(ENABLE_BACKEND_KINESIS)
|
||||
message(STATUS "kinesis backend: enabled")
|
||||
list(APPEND NETDATA_FILES ${KINESIS_BACKEND_FILES} ${KINESIS_EXPORTING_FILES})
|
||||
list(APPEND NETDATA_COMMON_LIBRARIES aws-cpp-sdk-kinesis aws-cpp-sdk-core ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
|
||||
list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
|
||||
list(APPEND NETDATA_COMMON_LIBRARIES ${KINESIS_LIBRARIES} ${AWS_CORE_LIBRARIES}
|
||||
${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
|
||||
list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${KINESIS_INCLUDE_DIRS} ${AWS_CORE_INCLUDE_DIRS}
|
||||
${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
|
||||
list(APPEND NETDATA_COMMON_CFLAGS ${CRYPTO_CFLAGS_OTHER} ${SSL_CFLAGS_OTHER} ${CURL_CFLAGS_OTHER})
|
||||
ELSE()
|
||||
message(STATUS "kinesis backend: disabled (requires AWS SDK for C++)")
|
||||
|
|
87
configure.ac
87
configure.ac
|
@ -1074,46 +1074,45 @@ PKG_CHECK_MODULES(
|
|||
[have_libcurl=no]
|
||||
)
|
||||
|
||||
AC_CHECK_LIB(
|
||||
PKG_CHECK_MODULES(
|
||||
[AWS_CPP_SDK_CORE],
|
||||
[aws-cpp-sdk-core],
|
||||
[cJSON_free],
|
||||
[have_libaws_cpp_sdk_core=yes],
|
||||
[have_libaws_cpp_sdk_core=no],
|
||||
[${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}]
|
||||
[AC_CHECK_LIB(
|
||||
[aws-cpp-sdk-core],
|
||||
[cJSON_free],
|
||||
[have_libaws_cpp_sdk_core=yes],
|
||||
[have_libaws_cpp_sdk_core=no]
|
||||
)],
|
||||
[have_libaws_cpp_sdk_core=no]
|
||||
)
|
||||
|
||||
AC_MSG_CHECKING([for Aws::Kinesis::Model::PutRecordRequest in -laws-cpp-sdk-kinesis])
|
||||
PKG_CHECK_MODULES(
|
||||
[AWS_CPP_SDK_KINESIS],
|
||||
[aws-cpp-sdk-kinesis],
|
||||
[have_libaws_cpp_sdk_kinesis=yes],
|
||||
[have_libaws_cpp_sdk_kinesis=no]
|
||||
)
|
||||
|
||||
if test "${have_libaws_cpp_sdk_core}" = "yes" -a "${have_libcrypto}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcurl}" = "yes"; then
|
||||
AC_LANG_SAVE
|
||||
AC_LANG_CPLUSPLUS
|
||||
save_LIBS="${LIBS}"
|
||||
LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
|
||||
save_CXXFLAGS="${CXXFLAGS}"
|
||||
CXXFLAGS="${CXXFLAGS} -std=c++11"
|
||||
AC_CHECK_LIB(
|
||||
[aws-checksums],
|
||||
[aws_checksums_crc32],
|
||||
[have_libaws_checksums=yes],
|
||||
[have_libaws_checksums=no]
|
||||
)
|
||||
|
||||
AC_TRY_LINK(
|
||||
[
|
||||
#include <aws/core/Aws.h>
|
||||
#include <aws/core/client/ClientConfiguration.h>
|
||||
#include <aws/core/auth/AWSCredentials.h>
|
||||
#include <aws/core/utils/Outcome.h>
|
||||
#include <aws/kinesis/KinesisClient.h>
|
||||
#include <aws/kinesis/model/PutRecordRequest.h>
|
||||
],
|
||||
[Aws::Kinesis::Model::PutRecordRequest request;],
|
||||
[have_libaws_cpp_sdk_kinesis=yes],
|
||||
[have_libaws_cpp_sdk_kinesis=no]
|
||||
)
|
||||
AC_CHECK_LIB(
|
||||
[aws-c-common],
|
||||
[aws_default_allocator],
|
||||
[have_libaws_c_common=yes],
|
||||
[have_libaws_c_common=no]
|
||||
)
|
||||
|
||||
LIBS="${save_LIBS}"
|
||||
CXXFLAGS="${save_CXXFLAGS}"
|
||||
AC_LANG_RESTORE
|
||||
else
|
||||
have_libaws_cpp_sdk_kinesis=no
|
||||
fi
|
||||
|
||||
AC_MSG_RESULT([${have_libaws_cpp_sdk_kinesis}])
|
||||
AC_CHECK_LIB(
|
||||
[aws-c-event-stream],
|
||||
[aws_event_stream_library_init],
|
||||
[have_libaws_c_event_stream=yes],
|
||||
[have_libaws_c_event_stream=no]
|
||||
)
|
||||
|
||||
test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \
|
||||
AC_MSG_ERROR([libaws-cpp-sdk-kinesis required but not found. try installing AWS C++ SDK])
|
||||
|
@ -1130,14 +1129,26 @@ test "${enable_backend_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \
|
|||
test "${enable_backend_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \
|
||||
AC_MSG_ERROR([libcrypto required but not found])
|
||||
|
||||
test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_checksums}" != "yes" \
|
||||
-a "${have_libaws_c_common}" != "yes" \
|
||||
-a "${have_libaws_c_event_stream}" != "yes" && \
|
||||
AC_MSG_ERROR([AWS SKD third party dependencies required but not found])
|
||||
|
||||
AC_MSG_CHECKING([if kinesis backend should be enabled])
|
||||
if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" = "yes" \
|
||||
-a "${have_libcurl}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcrypto}" = "yes"; then
|
||||
if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" \
|
||||
-a "${have_libaws_cpp_sdk_core}" = "yes" \
|
||||
-a "${have_libaws_checksums}" = "yes" \
|
||||
-a "${have_libaws_c_common}" = "yes" \
|
||||
-a "${have_libaws_c_event_stream}" = "yes" \
|
||||
-a "${have_libcurl}" = "yes" \
|
||||
-a "${have_libssl}" = "yes" \
|
||||
-a "${have_libcrypto}" = "yes"; then
|
||||
enable_backend_kinesis="yes"
|
||||
AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability])
|
||||
OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}"
|
||||
CXX11FLAG="-std=c++11"
|
||||
OPTIONAL_KINESIS_LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
|
||||
CXX11FLAG="${AWS_CPP_SDK_KINESIS_CFLAGS} ${AWS_CPP_SDK_CORE_CFLAGS}"
|
||||
OPTIONAL_KINESIS_LIBS="${AWS_CPP_SDK_KINESIS_LIBS} ${AWS_CPP_SDK_CORE_LIBS} \
|
||||
${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
|
||||
else
|
||||
enable_backend_kinesis="no"
|
||||
fi
|
||||
|
|
|
@ -10,20 +10,23 @@ sidebar_label: AWS Kinesis Data Streams
|
|||
## Prerequisites
|
||||
|
||||
To use AWS Kinesis for metric collecting and processing, you should first
|
||||
[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++. Netdata
|
||||
works with the SDK version 1.7.121. Other versions might work correctly as well, but they were not tested with Netdata.
|
||||
[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++.
|
||||
`libcrypto`, `libssl`, and `libcurl` are also required to compile Netdata with Kinesis support enabled. Next, Netdata
|
||||
should be re-installed from the source. The installer will detect that the required libraries are now available.
|
||||
|
||||
If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY="kinesis"`. Otherwise, the
|
||||
building process could take a very long time. Note that the default installation path for the libraries is
|
||||
If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY=kinesis`. Otherwise, the
|
||||
build process could take a very long time. Note, that the default installation path for the libraries is
|
||||
`/usr/local/lib64`. Many Linux distributions don't include this path as the default one for a library search, so it is
|
||||
advisable to use the following options to `cmake` while building the AWS SDK:
|
||||
|
||||
```sh
|
||||
cmake -DCMAKE_INSTALL_LIBDIR=/usr/lib -DCMAKE_INSTALL_INCLUDEDIR=/usr/include -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
|
||||
sudo cmake -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
|
||||
```
|
||||
|
||||
The `-DCMAKE_INSTALL_PREFIX=/usr` option also ensures that
|
||||
[third party dependencies](https://github.com/aws/aws-sdk-cpp#third-party-dependencies) are installed in your system
|
||||
during the SDK build process.
|
||||
|
||||
## Configuration
|
||||
|
||||
To enable data sending to the Kinesis service, run `./edit-config exporting.conf` in the Netdata configuration directory
|
||||
|
|
Loading…
Reference in New Issue