From 157fb5860a61c4472c2bc3ba11f2dd8712d9c845 Mon Sep 17 00:00:00 2001 From: Vladimir Kobal Date: Wed, 21 Apr 2021 10:50:06 +0300 Subject: [PATCH] Backend chart filtering backward compatibility fix (#11002) --- backends/backends.c | 6 +++++- backends/backends.h | 1 + database/rrd.h | 8 +++++--- database/rrdset.c | 4 ++++ exporting/check_filters.c | 8 ++++---- exporting/exporting_engine.h | 2 ++ exporting/prometheus/prometheus.c | 12 ++++++------ exporting/read_config.c | 8 ++++++-- exporting/tests/exporting_fixtures.c | 2 ++ exporting/tests/test_exporting_engine.c | 9 +++++---- 10 files changed, 40 insertions(+), 20 deletions(-) diff --git a/backends/backends.c b/backends/backends.c index eac933db44..dca21ef1c5 100644 --- a/backends/backends.c +++ b/backends/backends.c @@ -26,6 +26,7 @@ // const char *global_backend_prefix = "netdata"; +const char *global_backend_send_charts_matching = "*"; int global_backend_update_every = 10; BACKEND_OPTIONS global_backend_options = BACKEND_SOURCE_DATA_AVERAGE | BACKEND_OPTION_SEND_NAMES; const char *global_backend_source = NULL; @@ -517,7 +518,10 @@ void *backends_main(void *ptr) { else global_backend_options &= ~BACKEND_OPTION_SEND_NAMES; - charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT); + charts_pattern = simple_pattern_create( + global_backend_send_charts_matching = config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), + NULL, + SIMPLE_PATTERN_EXACT); hosts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT); #if ENABLE_PROMETHEUS_REMOTE_WRITE diff --git a/backends/backends.h b/backends/backends.h index 2f4efd90d7..77c58c9e41 100644 --- a/backends/backends.h +++ b/backends/backends.h @@ -37,6 +37,7 @@ extern int global_backend_update_every; extern BACKEND_OPTIONS global_backend_options; extern const char *global_backend_source; extern const char *global_backend_prefix; +extern const char *global_backend_send_charts_matching; extern void *backends_main(void *ptr); BACKEND_TYPE backend_select_type(const char *type); diff --git a/database/rrd.h b/database/rrd.h index 24a5e8e5db..ce0ef44821 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -454,8 +454,8 @@ typedef enum rrdset_flags { // (the master data set should be the one that has the same family and is not detail) RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete - RRDSET_FLAG_BACKEND_SEND = 1 << 4, // if set, this chart should be sent to backends - RRDSET_FLAG_BACKEND_IGNORE = 1 << 5, // if set, this chart should not be sent to backends + RRDSET_FLAG_EXPORTING_SEND = 1 << 4, // if set, this chart should be sent to Prometheus web API + RRDSET_FLAG_EXPORTING_IGNORE = 1 << 5, // if set, this chart should not be sent to Prometheus web API RRDSET_FLAG_UPSTREAM_SEND = 1 << 6, // if set, this chart should be sent upstream (streaming) RRDSET_FLAG_UPSTREAM_IGNORE = 1 << 7, // if set, this chart should not be sent upstream (streaming) RRDSET_FLAG_UPSTREAM_EXPOSED = 1 << 8, // if set, we have sent this chart definition to netdata parent (streaming) @@ -468,7 +468,9 @@ typedef enum rrdset_flags { // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at // least rrdset_free_obsolete_time seconds ago. RRDSET_FLAG_ARCHIVED = 1 << 15, - RRDSET_FLAG_ACLK = 1 << 16 + RRDSET_FLAG_ACLK = 1 << 16, + RRDSET_FLAG_BACKEND_SEND = 1 << 17, // if set, this chart should be sent to backends + RRDSET_FLAG_BACKEND_IGNORE = 1 << 18 // if set, this chart should not be sent to backends } RRDSET_FLAGS; #ifdef HAVE_C___ATOMIC diff --git a/database/rrdset.c b/database/rrdset.c index 15640d3ed8..da581c0cd5 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -175,6 +175,8 @@ int rrdset_set_name(RRDSET *st, const char *name) { if(unlikely(rrdset_index_add_name(host, st) != st)) error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name); + rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND); + rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE); rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_SEND); rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_IGNORE); rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND); @@ -858,6 +860,8 @@ RRDSET *rrdset_create_custom( rrdset_flag_clear(st, RRDSET_FLAG_DETAIL); rrdset_flag_clear(st, RRDSET_FLAG_DEBUG); rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); + rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND); + rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE); rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_SEND); rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_IGNORE); rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND); diff --git a/exporting/check_filters.c b/exporting/check_filters.c index 8d70c6f68d..64ced7238c 100644 --- a/exporting/check_filters.c +++ b/exporting/check_filters.c @@ -50,15 +50,15 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st) RRDSET_FLAGS *flags = &st->exporting_flags[instance->index]; - if(unlikely(*flags & RRDSET_FLAG_BACKEND_IGNORE)) + if(unlikely(*flags & RRDSET_FLAG_EXPORTING_IGNORE)) return 0; - if(unlikely(!(*flags & RRDSET_FLAG_BACKEND_SEND))) { + if(unlikely(!(*flags & RRDSET_FLAG_EXPORTING_SEND))) { // we have not checked this chart if(simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name)) - *flags |= RRDSET_FLAG_BACKEND_SEND; + *flags |= RRDSET_FLAG_EXPORTING_SEND; else { - *flags |= RRDSET_FLAG_BACKEND_IGNORE; + *flags |= RRDSET_FLAG_EXPORTING_IGNORE; debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, host->hostname); return 0; } diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h index 1d9feb7dd4..1edd283f78 100644 --- a/exporting/exporting_engine.h +++ b/exporting/exporting_engine.h @@ -77,6 +77,8 @@ struct instance_config { SIMPLE_PATTERN *charts_pattern; SIMPLE_PATTERN *hosts_pattern; + int initialized; + void *connector_specific_config; }; diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c index c10d94b90a..4222dce568 100644 --- a/exporting/prometheus/prometheus.c +++ b/exporting/prometheus/prometheus.c @@ -18,16 +18,16 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st) { RRDHOST *host = st->rrdhost; - if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE))) + if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_EXPORTING_IGNORE))) return 0; - if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_BACKEND_SEND))) { + if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_EXPORTING_SEND))) { // we have not checked this chart if (simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name)) - rrdset_flag_set(st, RRDSET_FLAG_BACKEND_SEND); + rrdset_flag_set(st, RRDSET_FLAG_EXPORTING_SEND); else { - rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE); + rrdset_flag_set(st, RRDSET_FLAG_EXPORTING_IGNORE); debug( D_BACKEND, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", @@ -855,7 +855,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host( EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options) { - if (unlikely(!prometheus_exporter_instance)) + if (unlikely(!prometheus_exporter_instance || !prometheus_exporter_instance->config.initialized)) return; prometheus_exporter_instance->before = now_realtime_sec(); @@ -892,7 +892,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts( EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options) { - if (unlikely(!prometheus_exporter_instance)) + if (unlikely(!prometheus_exporter_instance || !prometheus_exporter_instance->config.initialized)) return; prometheus_exporter_instance->before = now_realtime_sec(); diff --git a/exporting/read_config.c b/exporting/read_config.c index 995ba578f8..ea50fa0f6f 100644 --- a/exporting/read_config.c +++ b/exporting/read_config.c @@ -267,12 +267,16 @@ struct engine *read_exporting_config() else prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS; - prometheus_exporter_instance->config.charts_pattern = - simple_pattern_create(prometheus_config_get("send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT); + prometheus_exporter_instance->config.charts_pattern = simple_pattern_create( + prometheus_config_get("send charts matching", global_backend_send_charts_matching), + NULL, + SIMPLE_PATTERN_EXACT); prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create( prometheus_config_get("send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT); prometheus_exporter_instance->config.prefix = prometheus_config_get("prefix", global_backend_prefix); + + prometheus_exporter_instance->config.initialized = 1; } // TODO: change BACKEND to EXPORTING diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c index 00bb0ed0f2..b5b0ce8165 100644 --- a/exporting/tests/exporting_fixtures.c +++ b/exporting/tests/exporting_fixtures.c @@ -146,6 +146,8 @@ int setup_prometheus(void **state) prometheus_exporter_instance->config.charts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT); prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT); + prometheus_exporter_instance->config.initialized = 1; + return 0; } diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c index 774d1a2659..beb5691d8f 100644 --- a/exporting/tests/test_exporting_engine.c +++ b/exporting/tests/test_exporting_engine.c @@ -17,6 +17,7 @@ char log_line[MAX_LOG_LINE + 1]; BACKEND_OPTIONS global_backend_options = 0; const char *global_backend_source = "average"; const char *global_backend_prefix = "netdata"; +const char *global_backend_send_charts_matching = "*"; void init_connectors_in_tests(struct engine *engine) { @@ -268,7 +269,7 @@ static void test_rrdset_is_exportable(void **state) assert_int_equal(__real_rrdset_is_exportable(instance, st), 1); assert_ptr_not_equal(st->exporting_flags, NULL); - assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_BACKEND_SEND); + assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_EXPORTING_SEND); } static void test_false_rrdset_is_exportable(void **state) @@ -285,7 +286,7 @@ static void test_false_rrdset_is_exportable(void **state) assert_int_equal(__real_rrdset_is_exportable(instance, st), 0); assert_ptr_not_equal(st->exporting_flags, NULL); - assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_BACKEND_IGNORE); + assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_EXPORTING_IGNORE); } static void test_exporting_calculate_value_from_stored_data(void **state) @@ -993,9 +994,9 @@ static void test_can_send_rrdset(void **state) assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 1); - rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_BACKEND_IGNORE); + rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_EXPORTING_IGNORE); assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 0); - rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_BACKEND_IGNORE); + rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_EXPORTING_IGNORE); // TODO: test with a denying simple pattern