diff --git a/CMakeLists.txt b/CMakeLists.txt index 8143e5b694..82729885c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1498,6 +1498,7 @@ if(BUILD_TESTING) exporting/tests/netdata_doubles.c exporting/tests/system_doubles.c database/rrdlabels.c + database/rrdvar.c ) set(TEST_NAME exporting_engine) set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS) diff --git a/Makefile.am b/Makefile.am index 153731f978..e56a91f4fe 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1252,6 +1252,7 @@ if ENABLE_UNITTESTS $(EXPORTING_ENGINE_FILES) \ $(LIBNETDATA_FILES) \ database/rrdlabels.c \ + database/rrdvar.c \ $(NULL) exporting_tests_exporting_engine_testdriver_CFLAGS = \ $(AM_CFLAGS) \ diff --git a/aclk/aclk.c b/aclk/aclk.c index cfd85ad3a3..4b8750064a 100644 --- a/aclk/aclk.c +++ b/aclk/aclk.c @@ -762,7 +762,7 @@ void aclk_host_state_update(RRDHOST *host, int cmd) node_instance_creation_t node_instance_creation = { .claim_id = localhost->aclk_state.claimed_id, .hops = host->system_info->hops, - .hostname = host->hostname, + .hostname = rrdhost_hostname(host), .machine_guid = host->machine_guid }; create_query->data.bin_payload.payload = generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation); @@ -829,7 +829,7 @@ void aclk_send_node_instances() char host_id[UUID_STR_LEN]; uuid_unparse_lower(list->host_id, host_id); - RRDHOST *host = rrdhost_find_by_guid(host_id, 0); + RRDHOST *host = rrdhost_find_by_guid(host_id); struct capability caps[] = { { .name = "proto", .version = 1, .enabled = 1 }, { .name = "ml", .version = ml_capable(localhost), .enabled = host ? ml_enabled(host) : 0 }, @@ -867,7 +867,7 @@ void aclk_send_node_instances() rrdhost_aclk_state_unlock(localhost); info("Queuing registration for host=%s, hops=%d",(char*)node_instance_creation.machine_guid, list->hops); - freez(node_instance_creation.machine_guid); + freez((void *)node_instance_creation.machine_guid); aclk_queue_query(create_query); } freez(list->hostname); @@ -992,7 +992,7 @@ char *aclk_state(void) RRDHOST *host; rrd_rdlock(); rrdhost_foreach_read(host) { - buffer_sprintf(wb, "\n\n> Node Instance for mGUID: \"%s\" hostname \"%s\"\n", host->machine_guid, host->hostname); + buffer_sprintf(wb, "\n\n> Node Instance for mGUID: \"%s\" hostname \"%s\"\n", host->machine_guid, rrdhost_hostname(host)); buffer_strcat(wb, "\tClaimed ID: "); rrdhost_aclk_state_lock(host); @@ -1179,7 +1179,7 @@ char *aclk_state_json(void) rrdhost_foreach_read(host) { json_object *nodeinstance = json_object_new_object(); - tmp = json_object_new_string(host->hostname); + tmp = json_object_new_string(rrdhost_hostname(host)); json_object_object_add(nodeinstance, "hostname", tmp); tmp = json_object_new_string(host->machine_guid); diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c index 981c01965a..132d5fe18f 100644 --- a/aclk/aclk_query.c +++ b/aclk/aclk_query.c @@ -38,9 +38,7 @@ static RRDHOST *node_id_2_rrdhost(const char *node_id) int res; uuid_t node_id_bin, host_id_bin; - rrd_rdlock(); - RRDHOST *host = find_host_by_node_id((char *) node_id); - rrd_unlock(); + RRDHOST *host = find_host_by_node_id((char *)node_id); if (host) return host; @@ -54,7 +52,7 @@ static RRDHOST *node_id_2_rrdhost(const char *node_id) return NULL; } uuid_unparse_lower(host_id_bin, host_id); - return rrdhost_find_by_guid(host_id, 0); + return rrdhost_find_by_guid(host_id); } #define NODE_ID_QUERY "/node/" diff --git a/aclk/aclk_rx_msgs.c b/aclk/aclk_rx_msgs.c index e6ed332cc5..b42f010f9e 100644 --- a/aclk/aclk_rx_msgs.c +++ b/aclk/aclk_rx_msgs.c @@ -274,7 +274,7 @@ int create_node_instance_result(const char *msg, size_t msg_len) .node_id = res.node_id }; - RRDHOST *host = rrdhost_find_by_guid(res.machine_guid, 0); + RRDHOST *host = rrdhost_find_by_guid(res.machine_guid); if (host) { // not all host must have RRDHOST struct created for them // if they never connected during runtime of agent @@ -527,7 +527,7 @@ unsigned int aclk_init_rx_msg_handlers(void) return i; } -void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t msg_len, const char *topic) +void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t msg_len, const char *topic __maybe_unused) { if (aclk_stats_enabled) { ACLK_STATS_LOCK; diff --git a/aclk/schema-wrappers/node_creation.h b/aclk/schema-wrappers/node_creation.h index 190ccb4d6e..7a8c7f7c76 100644 --- a/aclk/schema-wrappers/node_creation.h +++ b/aclk/schema-wrappers/node_creation.h @@ -8,9 +8,9 @@ extern "C" { #endif typedef struct { - char* claim_id; - char* machine_guid; - char* hostname; + const char *claim_id; + const char *machine_guid; + const char *hostname; int32_t hops; } node_instance_creation_t; diff --git a/aclk/schema-wrappers/node_info.h b/aclk/schema-wrappers/node_info.h index e8ac2d7c60..de4ade78aa 100644 --- a/aclk/schema-wrappers/node_info.h +++ b/aclk/schema-wrappers/node_info.h @@ -19,41 +19,27 @@ struct machine_learning_info { }; struct aclk_node_info { - char *name; - - char *os; - char *os_name; - char *os_version; - - char *kernel_name; - char *kernel_version; - - char *architecture; + const char *name; + const char *os; + const char *os_name; + const char *os_version; + const char *kernel_name; + const char *kernel_version; + const char *architecture; uint32_t cpus; - - char *cpu_frequency; - - char *memory; - - char *disk_space; - - char *version; - - char *release_channel; - - char *timezone; - - char *virtualization_type; - - char *container_type; - - char *custom_info; - - char *machine_guid; + const char *cpu_frequency; + const char *memory; + const char *disk_space; + const char *version; + const char *release_channel; + const char *timezone; + const char *virtualization_type; + const char *container_type; + const char *custom_info; + const char *machine_guid; DICTIONARY *host_labels_ptr; - struct machine_learning_info ml_info; }; @@ -72,8 +58,8 @@ struct update_node_info { }; struct collector_info { - char *module; - char *plugin; + const char *module; + const char *plugin; }; struct update_node_collectors { diff --git a/collectors/cgroups.plugin/tests/test_doubles.c b/collectors/cgroups.plugin/tests/test_doubles.c index 6d85bc0f12..7f44416255 100644 --- a/collectors/cgroups.plugin/tests/test_doubles.c +++ b/collectors/cgroups.plugin/tests/test_doubles.c @@ -157,3 +157,8 @@ void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, cha void rrdcalc_update_rrdlabels(RRDSET *st) { (void)st; } + +void db_execute(const char *cmd) +{ + UNUSED(cmd); +} diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index 88e07fab7f..3f0a5ae8bb 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -105,13 +105,13 @@ PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char * if (rv) rrdvar_custom_host_variable_set(host, rv, value); else - error("cannot find/create HOST VARIABLE '%s' on host '%s'", name, host->hostname); + error("cannot find/create HOST VARIABLE '%s' on host '%s'", name, rrdhost_hostname(host)); } else { RRDSETVAR *rs = rrdsetvar_custom_chart_variable_create(st, name); if (rs) rrdsetvar_custom_chart_variable_set(rs, value); else - error("cannot find/create CHART VARIABLE '%s' on host '%s', chart '%s'", name, host->hostname, st->id); + error("cannot find/create CHART VARIABLE '%s' on host '%s', chart '%s'", name, rrdhost_hostname(host), rrdset_id(st)); } return PARSER_RC_OK; } @@ -184,7 +184,7 @@ PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, DICTIONARY *n { RRDSET *st = ((PARSER_USER_OBJECT *)user)->st; if (unlikely(!st)) { - error("requested CLABEL_COMMIT on host '%s', without a BEGIN, ignoring it.", host->hostname); + error("requested CLABEL_COMMIT on host '%s', without a BEGIN, ignoring it.", rrdhost_hostname(host)); return PARSER_RC_OK; } @@ -215,7 +215,7 @@ PARSER_RC pluginsd_set(char **words, void *user, PLUGINSD_ACTION *plugins_actio RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host; if (unlikely(!dimension || !*dimension)) { - error("requested a SET on chart '%s' of host '%s', without a dimension. Disabling it.", st->id, host->hostname); + error("requested a SET on chart '%s' of host '%s', without a dimension. Disabling it.", rrdset_id(st), rrdhost_hostname(host)); goto disable; } @@ -225,19 +225,19 @@ PARSER_RC pluginsd_set(char **words, void *user, PLUGINSD_ACTION *plugins_actio if (unlikely(!st)) { error( "requested a SET on dimension %s with value %s on host '%s', without a BEGIN. Disabling it.", dimension, - value ? value : "", host->hostname); + value ? value : "", rrdhost_hostname(host)); goto disable; } if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_PLUGINSD, "is setting dimension %s/%s to %s", st->id, dimension, value ? value : ""); + debug(D_PLUGINSD, "is setting dimension '%s'/'%s' to '%s'", rrdset_id(st), dimension, value ? value : ""); if (value) { RRDDIM *rd = rrddim_find(st, dimension); if (unlikely(!rd)) { error( "requested a SET to dimension with id '%s' on stats '%s' (%s) on host '%s', which does not exist. Disabling it.", - dimension, st->name, st->id, st->rrdhost->hostname); + dimension, rrdset_name(st), rrdset_id(st), rrdhost_hostname(st->rrdhost)); goto disable; } else { if (plugins_action->set_action) { @@ -262,13 +262,13 @@ PARSER_RC pluginsd_begin(char **words, void *user, PLUGINSD_ACTION *plugins_act RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host; if (unlikely(!id)) { - error("requested a BEGIN without a chart id for host '%s'. Disabling it.", host->hostname); + error("requested a BEGIN without a chart id for host '%s'. Disabling it.", rrdhost_hostname(host)); goto disable; } st = rrdset_find(host, id); if (unlikely(!st)) { - error("requested a BEGIN on chart '%s', which does not exist on host '%s'. Disabling it.", id, host->hostname); + error("requested a BEGIN on chart '%s', which does not exist on host '%s'. Disabling it.", id, rrdhost_hostname(host)); goto disable; } ((PARSER_USER_OBJECT *)user)->st = st; @@ -294,13 +294,13 @@ PARSER_RC pluginsd_end(char **words, void *user, PLUGINSD_ACTION *plugins_actio RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host; if (unlikely(!st)) { - error("requested an END, without a BEGIN on host '%s'. Disabling it.", host->hostname); + error("requested an END, without a BEGIN on host '%s'. Disabling it.", rrdhost_hostname(host)); ((PARSER_USER_OBJECT *) user)->enabled = 0; return PARSER_RC_ERROR; } if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_PLUGINSD, "requested an END on chart %s", st->id); + debug(D_PLUGINSD, "requested an END on chart '%s'", rrdset_id(st)); ((PARSER_USER_OBJECT *) user)->st = NULL; ((PARSER_USER_OBJECT *) user)->count++; @@ -343,7 +343,7 @@ PARSER_RC pluginsd_chart(char **words, void *user, PLUGINSD_ACTION *plugins_act // make sure we have the required variables if (unlikely((!type || !*type || !id || !*id))) { if (likely(host)) - error("requested a CHART, without a type.id, on host '%s'. Disabling it.", host->hostname); + error("requested a CHART, without a type.id, on host '%s'. Disabling it.", rrdhost_hostname(host)); else error("requested a CHART, without a type.id. Disabling it."); ((PARSER_USER_OBJECT *) user)->enabled = 0; @@ -424,13 +424,13 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION *plugins if (unlikely(!id)) { error( - "requested a DIMENSION, without an id, host '%s' and chart '%s'. Disabling it.", host->hostname, - st ? st->id : "UNSET"); + "requested a DIMENSION, without an id, host '%s' and chart '%s'. Disabling it.", rrdhost_hostname(host), + st ? rrdset_id(st) : "UNSET"); goto disable; } if (unlikely(!st && !((PARSER_USER_OBJECT *) user)->st_exists)) { - error("requested a DIMENSION, without a CHART, on host '%s'. Disabling it.", host->hostname); + error("requested a DIMENSION, without a CHART, on host '%s'. Disabling it.", rrdhost_hostname(host)); goto disable; } @@ -455,7 +455,7 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION *plugins debug( D_PLUGINSD, "creating dimension in chart %s, id='%s', name='%s', algorithm='%s', multiplier=%ld, divisor=%ld, hidden='%s'", - st->id, id, name ? name : "", rrd_algorithm_name(rrd_algorithm_id(algorithm)), multiplier, divisor, + rrdset_id(st), id, name ? name : "", rrd_algorithm_name(rrd_algorithm_id(algorithm)), multiplier, divisor, options ? options : ""); if (plugins_action->dimension_action) { @@ -494,7 +494,7 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_ } if (unlikely(!name || !*name)) { - error("requested a VARIABLE on host '%s', without a variable name. Disabling it.", host->hostname); + error("requested a VARIABLE on host '%s', without a variable name. Disabling it.", rrdhost_hostname(host)); ((PARSER_USER_OBJECT *)user)->enabled = 0; return PARSER_RC_ERROR; } @@ -504,12 +504,12 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_ if (unlikely(!value)) { error("cannot set %s VARIABLE '%s' on host '%s' to an empty value", (global) ? "HOST" : "CHART", name, - host->hostname); + rrdhost_hostname(host)); return PARSER_RC_OK; } if (!global && !st) { - error("cannot find/create CHART VARIABLE '%s' on host '%s' without a chart", name, host->hostname); + error("cannot find/create CHART VARIABLE '%s' on host '%s' without a chart", name, rrdhost_hostname(host)); return PARSER_RC_OK; } @@ -519,10 +519,10 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_ if (endptr == value) error( "the value '%s' of VARIABLE '%s' on host '%s' cannot be parsed as a number", value, name, - host->hostname); + rrdhost_hostname(host)); else error( - "the value '%s' of VARIABLE '%s' on host '%s' has leftovers: '%s'", value, name, host->hostname, + "the value '%s' of VARIABLE '%s' on host '%s' has leftovers: '%s'", value, name, rrdhost_hostname(host), endptr); } diff --git a/collectors/proc.plugin/ipc.c b/collectors/proc.plugin/ipc.c index b5c9ae5e14..e114a05a41 100644 --- a/collectors/proc.plugin/ipc.c +++ b/collectors/proc.plugin/ipc.c @@ -499,11 +499,11 @@ int do_ipc(int update_every, usec_t dt) { do_msg = CONFIG_BOOLEAN_NO; } else if(unlikely(!message_queue_root)) { - info("Making chart %s (%s) obsolete since it does not have any dimensions", st_msq_messages->name, st_msq_messages->id); + info("Making chart %s (%s) obsolete since it does not have any dimensions", rrdset_name(st_msq_messages), rrdset_id(st_msq_messages)); rrdset_is_obsolete(st_msq_messages); st_msq_messages = NULL; - info("Making chart %s (%s) obsolete since it does not have any dimensions", st_msq_bytes->name, st_msq_bytes->id); + info("Making chart %s (%s) obsolete since it does not have any dimensions", rrdset_name(st_msq_bytes), rrdset_id(st_msq_bytes)); rrdset_is_obsolete(st_msq_bytes); st_msq_bytes = NULL; } diff --git a/collectors/proc.plugin/proc_interrupts.c b/collectors/proc.plugin/proc_interrupts.c index 46290554b1..804fab88a5 100644 --- a/collectors/proc.plugin/proc_interrupts.c +++ b/collectors/proc.plugin/proc_interrupts.c @@ -173,7 +173,7 @@ int do_proc_interrupts(int update_every, usec_t dt) { // some interrupt may have changed without changing the total number of lines // if the same number of interrupts have been added and removed between two // calls of this function. - if(unlikely(!irr->rd || strncmp(irr->rd->name, irr->name, MAX_INTERRUPT_NAME) != 0)) { + if(unlikely(!irr->rd || strncmp(rrddim_name(irr->rd), irr->name, MAX_INTERRUPT_NAME) != 0)) { irr->rd = rrddim_add(st_system_interrupts, irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_set_name(st_system_interrupts, irr->rd, irr->name); diff --git a/collectors/proc.plugin/proc_softirqs.c b/collectors/proc.plugin/proc_softirqs.c index 7eff28c989..feefa98855 100644 --- a/collectors/proc.plugin/proc_softirqs.c +++ b/collectors/proc.plugin/proc_softirqs.c @@ -153,7 +153,7 @@ int do_proc_softirqs(int update_every, usec_t dt) { // some interrupt may have changed without changing the total number of lines // if the same number of interrupts have been added and removed between two // calls of this function. - if(unlikely(!irr->rd || strncmp(irr->name, irr->rd->name, MAX_INTERRUPT_NAME) != 0)) { + if(unlikely(!irr->rd || strncmp(irr->name, rrddim_name(irr->rd), MAX_INTERRUPT_NAME) != 0)) { irr->rd = rrddim_add(st_system_softirqs, irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_set_name(st_system_softirqs, irr->rd, irr->name); diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c index fef4206bc6..e68640a4f7 100644 --- a/collectors/statsd.plugin/statsd.c +++ b/collectors/statsd.plugin/statsd.c @@ -2228,7 +2228,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch statsd_add_dim_to_app_chart(app, chart, dim); if (unlikely(dim->value_ptr)) { - debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr); + debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, rrddim_id(dim->rd), chart->id, rrdset_id(chart->st), app->name, *dim->value_ptr); rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr); } } diff --git a/daemon/analytics.c b/daemon/analytics.c index 370818b8a3..6c40930b75 100644 --- a/daemon/analytics.c +++ b/daemon/analytics.c @@ -10,8 +10,8 @@ extern void analytics_build_info (BUFFER *b); extern int aclk_connected; struct collector { - char *plugin; - char *module; + const char *plugin; + const char *module; }; struct array_printer { @@ -286,8 +286,10 @@ void analytics_collectors(void) rrdset_foreach_read(st, localhost) { if (rrdset_is_available_for_viewers(st)) { - struct collector col = { .plugin = st->plugin_name ? st->plugin_name : "", - .module = st->module_name ? st->module_name : "" }; + struct collector col = { + .plugin = rrdset_plugin_name(st), + .module = rrdset_module_name(st) + }; snprintfz(name, 499, "%s:%s", col.plugin, col.module); dictionary_set(dict, name, &col, sizeof(struct collector)); } @@ -441,7 +443,7 @@ void analytics_alarms(void) int alarm_warn = 0, alarm_crit = 0, alarm_normal = 0; char b[10]; RRDCALC *rc; - for (rc = localhost->alarms; rc; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(localhost, rc) { if (unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; @@ -539,7 +541,7 @@ void analytics_gather_mutable_meta_data(void) analytics_alarms_notifications(); analytics_set_data( - &analytics_data.netdata_config_is_parent, (localhost->next || configured_as_parent()) ? "true" : "false"); + &analytics_data.netdata_config_is_parent, (rrdhost_hosts_available() > 1 || configured_as_parent()) ? "true" : "false"); char *claim_id = get_agent_claimid(); analytics_set_data(&analytics_data.netdata_host_agent_claimed, claim_id ? "true" : "false"); diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index 9ef9ebc851..22fb0997a0 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -9,8 +9,9 @@ #define WORKER_JOB_WORKERS 2 #define WORKER_JOB_DBENGINE 3 #define WORKER_JOB_HEARTBEAT 4 +#define WORKER_JOB_STRINGS 5 -#if WORKER_UTILIZATION_MAX_JOB_TYPES < 5 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 6 #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5 #endif @@ -38,6 +39,10 @@ static struct global_statistics { volatile uint64_t sqlite3_queries_failed_locked; volatile uint64_t sqlite3_rows; + volatile uint64_t rrdcontext_metric_triggers; + volatile uint64_t rrdcontext_instance_triggers; + volatile uint64_t rrdcontext_context_triggers; + } global_statistics = { .connected_clients = 0, .web_requests = 0, @@ -80,6 +85,18 @@ void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_genera __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, result_points_generated, __ATOMIC_RELAXED); } +void rrdcontext_triggered_update_on_rrdmetric() { + __atomic_fetch_add(&global_statistics.rrdcontext_metric_triggers, 1, __ATOMIC_RELAXED); +} + +void rrdcontext_triggered_update_on_rrdinstance() { + __atomic_fetch_add(&global_statistics.rrdcontext_instance_triggers, 1, __ATOMIC_RELAXED); +} + +void rrdcontext_triggered_update_on_rrdcontext() { + __atomic_fetch_add(&global_statistics.rrdcontext_context_triggers, 1, __ATOMIC_RELAXED); +} + void finished_web_request_statistics(uint64_t dt, uint64_t bytes_received, uint64_t bytes_sent, @@ -133,6 +150,10 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t gs->sqlite3_queries_failed_busy = __atomic_load_n(&global_statistics.sqlite3_queries_failed_busy, __ATOMIC_RELAXED); gs->sqlite3_queries_failed_locked = __atomic_load_n(&global_statistics.sqlite3_queries_failed_locked, __ATOMIC_RELAXED); gs->sqlite3_rows = __atomic_load_n(&global_statistics.sqlite3_rows, __ATOMIC_RELAXED); + + gs->rrdcontext_metric_triggers = __atomic_load_n(&global_statistics.rrdcontext_metric_triggers, __ATOMIC_RELAXED); + gs->rrdcontext_instance_triggers = __atomic_load_n(&global_statistics.rrdcontext_instance_triggers, __ATOMIC_RELAXED); + gs->rrdcontext_context_triggers = __atomic_load_n(&global_statistics.rrdcontext_context_triggers, __ATOMIC_RELAXED); } static void global_statistics_charts(void) { @@ -579,6 +600,42 @@ static void global_statistics_charts(void) { rrdset_done(st_sqlite3_rows); } + + // ---------------------------------------------------------------- + + if(gs.rrdcontext_context_triggers || gs.rrdcontext_instance_triggers || gs.rrdcontext_metric_triggers) { + static RRDSET *st_rrdcontext_triggers = NULL; + static RRDDIM *rd_context = NULL, *rd_instance = NULL, *rd_metric = NULL; + + if (unlikely(!st_rrdcontext_triggers)) { + st_rrdcontext_triggers = rrdset_create_localhost( + "netdata" + , "rrdcontext_triggers" + , NULL + , "rrdcontext" + , NULL + , "Netdata RRD context triggers" + , "triggers/s" + , "netdata" + , "stats" + , 131200 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_metric = rrddim_add(st_rrdcontext_triggers, "metric", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_instance = rrddim_add(st_rrdcontext_triggers, "instance", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_context = rrddim_add(st_rrdcontext_triggers, "context", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_rrdcontext_triggers); + + rrddim_set_by_pointer(st_rrdcontext_triggers, rd_metric, (collected_number)gs.rrdcontext_metric_triggers); + rrddim_set_by_pointer(st_rrdcontext_triggers, rd_instance, (collected_number)gs.rrdcontext_instance_triggers); + rrddim_set_by_pointer(st_rrdcontext_triggers, rd_context, (collected_number)gs.rrdcontext_context_triggers); + + rrdset_done(st_rrdcontext_triggers); + } } static void dbengine_statistics_charts(void) { @@ -988,6 +1045,93 @@ static void dbengine_statistics_charts(void) { #endif } +static void update_strings_charts() { + static RRDSET *st_ops = NULL, *st_entries = NULL, *st_mem = NULL; + static RRDDIM *rd_ops_inserts = NULL, *rd_ops_deletes = NULL, *rd_ops_searches = NULL, *rd_ops_duplications = NULL, *rd_ops_releases = NULL; + static RRDDIM *rd_entries_entries = NULL, *rd_entries_refs = NULL; + static RRDDIM *rd_mem = NULL; + + size_t inserts, deletes, searches, entries, references, memory, duplications, releases; + + string_statistics(&inserts, &deletes, &searches, &entries, &references, &memory, &duplications, &releases); + + if (unlikely(!st_ops)) { + st_ops = rrdset_create_localhost( + "netdata" + , "strings_ops" + , NULL + , "strings" + , NULL + , "Strings operations" + , "ops/s" + , "netdata" + , "stats" + , 910000 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE); + + rd_ops_inserts = rrddim_add(st_ops, "inserts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ops_deletes = rrddim_add(st_ops, "deletes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ops_searches = rrddim_add(st_ops, "searches", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ops_duplications = rrddim_add(st_ops, "duplications", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ops_releases = rrddim_add(st_ops, "releases", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(st_ops); + + rrddim_set_by_pointer(st_ops, rd_ops_inserts, (collected_number)inserts); + rrddim_set_by_pointer(st_ops, rd_ops_deletes, (collected_number)deletes); + rrddim_set_by_pointer(st_ops, rd_ops_searches, (collected_number)searches); + rrddim_set_by_pointer(st_ops, rd_ops_duplications, (collected_number)duplications); + rrddim_set_by_pointer(st_ops, rd_ops_releases, (collected_number)releases); + rrdset_done(st_ops); + + if (unlikely(!st_entries)) { + st_entries = rrdset_create_localhost( + "netdata" + , "strings_entries" + , NULL + , "strings" + , NULL + , "Strings entries" + , "entries" + , "netdata" + , "stats" + , 910001 + , localhost->rrd_update_every + , RRDSET_TYPE_AREA); + + rd_entries_entries = rrddim_add(st_entries, "entries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_entries_refs = rrddim_add(st_entries, "references", NULL, 1, -1, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(st_entries); + + rrddim_set_by_pointer(st_entries, rd_entries_entries, (collected_number)entries); + rrddim_set_by_pointer(st_entries, rd_entries_refs, (collected_number)references); + rrdset_done(st_entries); + + if (unlikely(!st_mem)) { + st_mem = rrdset_create_localhost( + "netdata" + , "strings_memory" + , NULL + , "strings" + , NULL + , "Strings memory" + , "bytes" + , "netdata" + , "stats" + , 910001 + , localhost->rrd_update_every + , RRDSET_TYPE_AREA); + + rd_mem = rrddim_add(st_mem, "memory", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(st_mem); + + rrddim_set_by_pointer(st_mem, rd_mem, (collected_number)memory); + rrdset_done(st_mem); +} + static void update_heartbeat_charts() { static RRDSET *st_heartbeat = NULL; static RRDDIM *rd_heartbeat_min = NULL; @@ -1032,13 +1176,26 @@ static void update_heartbeat_charts() { #define WORKERS_MIN_PERCENT_DEFAULT 10000.0 -struct worker_job_type { - char name[WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH + 1]; +struct worker_job_type_gs { + STRING *name; + STRING *units; + size_t jobs_started; usec_t busy_time; RRDDIM *rd_jobs_started; RRDDIM *rd_busy_time; + + WORKER_METRIC_TYPE type; + NETDATA_DOUBLE min_value; + NETDATA_DOUBLE max_value; + NETDATA_DOUBLE sum_value; + size_t count_value; + + RRDSET *st; + RRDDIM *rd_min; + RRDDIM *rd_max; + RRDDIM *rd_avg; }; struct worker_thread { @@ -1071,7 +1228,7 @@ struct worker_utilization { char *name_lowercase; - struct worker_job_type per_job_type[WORKER_UTILIZATION_MAX_JOB_TYPES]; + struct worker_job_type_gs per_job_type[WORKER_UTILIZATION_MAX_JOB_TYPES]; size_t workers_registered; size_t workers_busy; @@ -1170,14 +1327,16 @@ static void workers_total_cpu_utilization_chart(void) { if(!wu->workers_cpu_registered) continue; if(!wu->rd_total_cpu_utilizaton) - wu->rd_total_cpu_utilizaton = rrddim_add(st, wu->name_lowercase, NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE); + wu->rd_total_cpu_utilizaton = rrddim_add(st, wu->name_lowercase, NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); - rrddim_set_by_pointer(st, wu->rd_total_cpu_utilizaton, (collected_number)((double)wu->workers_cpu_total * 10000.0)); + rrddim_set_by_pointer(st, wu->rd_total_cpu_utilizaton, (collected_number)((double)wu->workers_cpu_total * 100.0)); } rrdset_done(st); } +#define WORKER_CHART_DECIMAL_PRECISION 100 + static void workers_utilization_update_chart(struct worker_utilization *wu) { if(!wu->workers_registered) return; @@ -1215,28 +1374,28 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) { // we add the min and max dimensions only when we have multiple workers if(unlikely(!wu->rd_workers_time_min && wu->workers_registered > 1)) - wu->rd_workers_time_min = rrddim_add(wu->st_workers_time, "min", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE); + wu->rd_workers_time_min = rrddim_add(wu->st_workers_time, "min", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); if(unlikely(!wu->rd_workers_time_max && wu->workers_registered > 1)) - wu->rd_workers_time_max = rrddim_add(wu->st_workers_time, "max", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE); + wu->rd_workers_time_max = rrddim_add(wu->st_workers_time, "max", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); if(unlikely(!wu->rd_workers_time_avg)) - wu->rd_workers_time_avg = rrddim_add(wu->st_workers_time, "average", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE); + wu->rd_workers_time_avg = rrddim_add(wu->st_workers_time, "average", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); rrdset_next(wu->st_workers_time); if(unlikely(wu->workers_min_busy_time == WORKERS_MIN_PERCENT_DEFAULT)) wu->workers_min_busy_time = 0.0; if(wu->rd_workers_time_min) - rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_min, (collected_number)((double)wu->workers_min_busy_time * 10000.0)); + rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_min, (collected_number)((double)wu->workers_min_busy_time * WORKER_CHART_DECIMAL_PRECISION)); if(wu->rd_workers_time_max) - rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_max, (collected_number)((double)wu->workers_max_busy_time * 10000.0)); + rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_max, (collected_number)((double)wu->workers_max_busy_time * WORKER_CHART_DECIMAL_PRECISION)); if(wu->workers_total_duration == 0) rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, 0); else - rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, (collected_number)((double)wu->workers_total_busy_time * 100.0 * 10000.0 / (double)wu->workers_total_duration)); + rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, (collected_number)((double)wu->workers_total_busy_time * 100.0 * WORKER_CHART_DECIMAL_PRECISION / (double)wu->workers_total_duration)); rrdset_done(wu->st_workers_time); @@ -1268,28 +1427,28 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) { } if (unlikely(!wu->rd_workers_cpu_min && wu->workers_registered > 1)) - wu->rd_workers_cpu_min = rrddim_add(wu->st_workers_cpu, "min", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE); + wu->rd_workers_cpu_min = rrddim_add(wu->st_workers_cpu, "min", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); if (unlikely(!wu->rd_workers_cpu_max && wu->workers_registered > 1)) - wu->rd_workers_cpu_max = rrddim_add(wu->st_workers_cpu, "max", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE); + wu->rd_workers_cpu_max = rrddim_add(wu->st_workers_cpu, "max", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); if(unlikely(!wu->rd_workers_cpu_avg)) - wu->rd_workers_cpu_avg = rrddim_add(wu->st_workers_cpu, "average", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE); + wu->rd_workers_cpu_avg = rrddim_add(wu->st_workers_cpu, "average", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); rrdset_next(wu->st_workers_cpu); if(unlikely(wu->workers_cpu_min == WORKERS_MIN_PERCENT_DEFAULT)) wu->workers_cpu_min = 0.0; if(wu->rd_workers_cpu_min) - rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_min, (collected_number)(wu->workers_cpu_min * 10000ULL)); + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_min, (collected_number)(wu->workers_cpu_min * WORKER_CHART_DECIMAL_PRECISION)); if(wu->rd_workers_cpu_max) - rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_max, (collected_number)(wu->workers_cpu_max * 10000ULL)); + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_max, (collected_number)(wu->workers_cpu_max * WORKER_CHART_DECIMAL_PRECISION)); if(wu->workers_cpu_registered == 0) rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, 0); else - rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (NETDATA_DOUBLE)wu->workers_cpu_registered )); + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * WORKER_CHART_DECIMAL_PRECISION / (NETDATA_DOUBLE)wu->workers_cpu_registered )); rrdset_done(wu->st_workers_cpu); } @@ -1325,10 +1484,13 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) { { size_t i; for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) { - if (wu->per_job_type[i].name[0]) { + if(unlikely(wu->per_job_type[i].type != WORKER_METRIC_IDLE_BUSY)) + continue; + + if (wu->per_job_type[i].name) { if(unlikely(!wu->per_job_type[i].rd_jobs_started)) - wu->per_job_type[i].rd_jobs_started = rrddim_add(wu->st_workers_jobs_per_job_type, wu->per_job_type[i].name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + wu->per_job_type[i].rd_jobs_started = rrddim_add(wu->st_workers_jobs_per_job_type, string2str(wu->per_job_type[i].name), NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_set_by_pointer(wu->st_workers_jobs_per_job_type, wu->per_job_type[i].rd_jobs_started, (collected_number)(wu->per_job_type[i].jobs_started)); } @@ -1367,10 +1529,13 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) { { size_t i; for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) { - if (wu->per_job_type[i].name[0]) { + if(unlikely(wu->per_job_type[i].type != WORKER_METRIC_IDLE_BUSY)) + continue; + + if (wu->per_job_type[i].name) { if(unlikely(!wu->per_job_type[i].rd_busy_time)) - wu->per_job_type[i].rd_busy_time = rrddim_add(wu->st_workers_busy_per_job_type, wu->per_job_type[i].name, NULL, 1, USEC_PER_MS, RRD_ALGORITHM_ABSOLUTE); + wu->per_job_type[i].rd_busy_time = rrddim_add(wu->st_workers_busy_per_job_type, string2str(wu->per_job_type[i].name), NULL, 1, USEC_PER_MS, RRD_ALGORITHM_ABSOLUTE); rrddim_set_by_pointer(wu->st_workers_busy_per_job_type, wu->per_job_type[i].rd_busy_time, (collected_number)(wu->per_job_type[i].busy_time)); } @@ -1414,6 +1579,122 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) { rrddim_set_by_pointer(wu->st_workers_threads, wu->rd_workers_threads_busy, (collected_number)(wu->workers_busy)); rrdset_done(wu->st_workers_threads); } + + // ---------------------------------------------------------------------- + // custom metric types WORKER_METRIC_ABSOLUTE + + { + size_t i; + for (i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES; i++) { + if(wu->per_job_type[i].type != WORKER_METRIC_ABSOLUTE) + continue; + + if(!wu->per_job_type[i].count_value) + continue; + + if(!wu->per_job_type[i].st) { + size_t job_name_len = string_strlen(wu->per_job_type[i].name); + if(job_name_len > RRD_ID_LENGTH_MAX) job_name_len = RRD_ID_LENGTH_MAX; + + char job_name_sanitized[job_name_len + 1]; + rrdset_strncpyz_name(job_name_sanitized, string2str(wu->per_job_type[i].name), job_name_len); + + char name[RRD_ID_LENGTH_MAX + 1]; + snprintfz(name, RRD_ID_LENGTH_MAX, "workers_%s_value_%s", wu->name_lowercase, job_name_sanitized); + + char context[RRD_ID_LENGTH_MAX + 1]; + snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.value.%s", wu->name_lowercase, job_name_sanitized); + + char title[1000 + 1]; + snprintf(title, 1000, "Netdata Workers %s Value of %s", wu->name_lowercase, string2str(wu->per_job_type[i].name)); + + wu->per_job_type[i].st = rrdset_create_localhost( + "netdata" + , name + , NULL + , wu->family + , context + , title + , (wu->per_job_type[i].units)?string2str(wu->per_job_type[i].units):"value" + , "netdata" + , "stats" + , wu->priority + 5 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + wu->per_job_type[i].rd_min = rrddim_add(wu->per_job_type[i].st, "min", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); + wu->per_job_type[i].rd_max = rrddim_add(wu->per_job_type[i].st, "max", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); + wu->per_job_type[i].rd_avg = rrddim_add(wu->per_job_type[i].st, "average", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(wu->per_job_type[i].st); + + rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_min, (collected_number)(wu->per_job_type[i].min_value * WORKER_CHART_DECIMAL_PRECISION)); + rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_max, (collected_number)(wu->per_job_type[i].max_value * WORKER_CHART_DECIMAL_PRECISION)); + rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_avg, (collected_number)(wu->per_job_type[i].sum_value / wu->per_job_type[i].count_value * WORKER_CHART_DECIMAL_PRECISION)); + + rrdset_done(wu->per_job_type[i].st); + } + } + + // ---------------------------------------------------------------------- + // custom metric types WORKER_METRIC_INCREMENTAL + + { + size_t i; + for (i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES; i++) { + if(wu->per_job_type[i].type != WORKER_METRIC_INCREMENTAL) + continue; + + if(!wu->per_job_type[i].count_value) + continue; + + if(!wu->per_job_type[i].st) { + size_t job_name_len = string_strlen(wu->per_job_type[i].name); + if(job_name_len > RRD_ID_LENGTH_MAX) job_name_len = RRD_ID_LENGTH_MAX; + + char job_name_sanitized[job_name_len + 1]; + rrdset_strncpyz_name(job_name_sanitized, string2str(wu->per_job_type[i].name), job_name_len); + + char name[RRD_ID_LENGTH_MAX + 1]; + snprintfz(name, RRD_ID_LENGTH_MAX, "workers_%s_rate_%s", wu->name_lowercase, job_name_sanitized); + + char context[RRD_ID_LENGTH_MAX + 1]; + snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.rate.%s", wu->name_lowercase, job_name_sanitized); + + char title[1000 + 1]; + snprintf(title, 1000, "Netdata Workers %s Rate of %s", wu->name_lowercase, string2str(wu->per_job_type[i].name)); + + wu->per_job_type[i].st = rrdset_create_localhost( + "netdata" + , name + , NULL + , wu->family + , context + , title + , (wu->per_job_type[i].units)?string2str(wu->per_job_type[i].units):"rate" + , "netdata" + , "stats" + , wu->priority + 5 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + wu->per_job_type[i].rd_min = rrddim_add(wu->per_job_type[i].st, "min", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); + wu->per_job_type[i].rd_max = rrddim_add(wu->per_job_type[i].st, "max", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); + wu->per_job_type[i].rd_avg = rrddim_add(wu->per_job_type[i].st, "average", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(wu->per_job_type[i].st); + + rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_min, (collected_number)(wu->per_job_type[i].min_value * WORKER_CHART_DECIMAL_PRECISION)); + rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_max, (collected_number)(wu->per_job_type[i].max_value * WORKER_CHART_DECIMAL_PRECISION)); + rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_avg, (collected_number)(wu->per_job_type[i].sum_value / wu->per_job_type[i].count_value * WORKER_CHART_DECIMAL_PRECISION)); + + rrdset_done(wu->per_job_type[i].st); + } + } } static void workers_utilization_reset_statistics(struct worker_utilization *wu) { @@ -1440,6 +1721,11 @@ static void workers_utilization_reset_statistics(struct worker_utilization *wu) wu->per_job_type[i].jobs_started = 0; wu->per_job_type[i].busy_time = 0; + + wu->per_job_type[i].min_value = NAN; + wu->per_job_type[i].max_value = NAN; + wu->per_job_type[i].sum_value = NAN; + wu->per_job_type[i].count_value = 0; } struct worker_thread *wt; @@ -1522,7 +1808,20 @@ static struct worker_thread *worker_thread_find_or_create(struct worker_utilizat return wt; } -static void worker_utilization_charts_callback(void *ptr, pid_t pid __maybe_unused, const char *thread_tag __maybe_unused, size_t utilization_usec __maybe_unused, size_t duration_usec __maybe_unused, size_t jobs_started __maybe_unused, size_t is_running __maybe_unused, const char **job_types_names __maybe_unused, size_t *job_types_jobs_started __maybe_unused, usec_t *job_types_busy_time __maybe_unused) { +static void worker_utilization_charts_callback(void *ptr + , pid_t pid __maybe_unused + , const char *thread_tag __maybe_unused + , size_t utilization_usec __maybe_unused + , size_t duration_usec __maybe_unused + , size_t jobs_started __maybe_unused + , size_t is_running __maybe_unused + , STRING **job_types_names __maybe_unused + , STRING **job_types_units __maybe_unused + , WORKER_METRIC_TYPE *job_types_metric_types __maybe_unused + , size_t *job_types_jobs_started __maybe_unused + , usec_t *job_types_busy_time __maybe_unused + , NETDATA_DOUBLE *job_types_custom_metrics __maybe_unused + ) { struct worker_utilization *wu = (struct worker_utilization *)ptr; // find the worker_thread in the list @@ -1552,12 +1851,32 @@ static void worker_utilization_charts_callback(void *ptr, pid_t pid __maybe_unus // accumulate per job type statistics size_t i; for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) { + if(!wu->per_job_type[i].name && job_types_names[i]) + wu->per_job_type[i].name = string_dup(job_types_names[i]); + + if(!wu->per_job_type[i].units && job_types_units[i]) + wu->per_job_type[i].units = string_dup(job_types_units[i]); + + wu->per_job_type[i].type = job_types_metric_types[i]; + wu->per_job_type[i].jobs_started += job_types_jobs_started[i]; wu->per_job_type[i].busy_time += job_types_busy_time[i]; - // new job type found - if(unlikely(!wu->per_job_type[i].name[0] && job_types_names[i])) - strncpyz(wu->per_job_type[i].name, job_types_names[i], WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH); + NETDATA_DOUBLE value = job_types_custom_metrics[i]; + if(netdata_double_isnumber(value)) { + if(!wu->per_job_type[i].count_value) { + wu->per_job_type[i].count_value = 1; + wu->per_job_type[i].min_value = value; + wu->per_job_type[i].max_value = value; + wu->per_job_type[i].sum_value = value; + } + else { + wu->per_job_type[i].count_value++; + wu->per_job_type[i].sum_value += value; + if(value < wu->per_job_type[i].min_value) wu->per_job_type[i].min_value = value; + if(value > wu->per_job_type[i].max_value) wu->per_job_type[i].max_value = value; + } + } } // find its CPU utilization @@ -1607,6 +1926,14 @@ static void worker_utilization_finish(void) { wu->name_lowercase = NULL; } + for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) { + string_freez(wu->per_job_type[i].name); + wu->per_job_type[i].name = NULL; + + string_freez(wu->per_job_type[i].units); + wu->per_job_type[i].units = NULL; + } + // mark all threads as not enabled struct worker_thread *t; for(t = wu->threads; t ; t = t->next) t->enabled = 0; @@ -1639,6 +1966,7 @@ void *global_statistics_main(void *ptr) worker_register_job_name(WORKER_JOB_REGISTRY, "registry"); worker_register_job_name(WORKER_JOB_WORKERS, "workers"); worker_register_job_name(WORKER_JOB_DBENGINE, "dbengine"); + worker_register_job_name(WORKER_JOB_STRINGS, "strings"); netdata_thread_cleanup_push(global_statistics_cleanup, ptr); @@ -1673,6 +2001,9 @@ void *global_statistics_main(void *ptr) worker_is_busy(WORKER_JOB_HEARTBEAT); update_heartbeat_charts(); + + worker_is_busy(WORKER_JOB_STRINGS); + update_strings_charts(); } netdata_thread_cleanup_pop(1); diff --git a/daemon/global_statistics.h b/daemon/global_statistics.h index 8d4a63d085..8f50a9087c 100644 --- a/daemon/global_statistics.h +++ b/daemon/global_statistics.h @@ -12,6 +12,10 @@ extern void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points extern void sqlite3_query_completed(bool success, bool busy, bool locked); extern void sqlite3_row_completed(void); +extern void rrdcontext_triggered_update_on_rrdmetric(void); +extern void rrdcontext_triggered_update_on_rrdinstance(void); +extern void rrdcontext_triggered_update_on_rrdcontext(void); + extern void finished_web_request_statistics(uint64_t dt, uint64_t bytes_received, uint64_t bytes_sent, diff --git a/daemon/unit_test.c b/daemon/unit_test.c index 33df4da357..a2cff95493 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -1197,12 +1197,12 @@ int run_test(struct test *test) fprintf(stderr, " > %s: feeding position %lu\n", test->name, c+1); } - fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rd->name, test->feed[c].value); + fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rrddim_name(rd), test->feed[c].value); rrddim_set(st, "dim1", test->feed[c].value); last = test->feed[c].value; if(rd2) { - fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rd2->name, test->feed2[c]); + fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rrddim_name(rd2), test->feed2[c]); rrddim_set(st, "dim2", test->feed2[c]); } @@ -1231,7 +1231,7 @@ int run_test(struct test *test) int same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0; fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT ", found " NETDATA_DOUBLE_FORMAT ", %s\n", - test->name, rd->name, c+1, + test->name, rrddim_name(rd), c+1, (int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start), n, v, (same)?"OK":"### E R R O R ###"); @@ -1243,7 +1243,7 @@ int run_test(struct test *test) same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0; fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT ", found " NETDATA_DOUBLE_FORMAT ", %s\n", - test->name, rd2->name, c+1, + test->name, rrddim_name(rd2), c+1, (int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start), n, v, (same)?"OK":"### E R R O R ###"); if(!same) errors++; @@ -1258,39 +1258,39 @@ static int test_variable_renames(void) { fprintf(stderr, "Creating chart\n"); RRDSET *st = rrdset_create_localhost("chart", "ID", NULL, "family", "context", "Unit Testing", "a value", "unittest", NULL, 1, 1, RRDSET_TYPE_LINE); - fprintf(stderr, "Created chart with id '%s', name '%s'\n", st->id, st->name); + fprintf(stderr, "Created chart with id '%s', name '%s'\n", rrdset_id(st), rrdset_name(st)); fprintf(stderr, "Creating dimension DIM1\n"); RRDDIM *rd1 = rrddim_add(st, "DIM1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rd1->id, rd1->name); + fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rrddim_id(rd1), rrddim_name(rd1)); fprintf(stderr, "Creating dimension DIM2\n"); RRDDIM *rd2 = rrddim_add(st, "DIM2", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rd2->id, rd2->name); + fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rrddim_id(rd2), rrddim_name(rd2)); fprintf(stderr, "Renaming chart to CHARTNAME1\n"); rrdset_set_name(st, "CHARTNAME1"); - fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", st->id, st->name); + fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", rrdset_id(st), rrdset_name(st)); fprintf(stderr, "Renaming chart to CHARTNAME2\n"); rrdset_set_name(st, "CHARTNAME2"); - fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", st->id, st->name); + fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", rrdset_id(st), rrdset_name(st)); fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME1\n"); rrddim_set_name(st, rd1, "DIM1NAME1"); - fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd1->id, rd1->name); + fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd1), rrddim_name(rd1)); fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME2\n"); rrddim_set_name(st, rd1, "DIM1NAME2"); - fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd1->id, rd1->name); + fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd1), rrddim_name(rd1)); fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME1\n"); rrddim_set_name(st, rd2, "DIM2NAME1"); - fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd2->id, rd2->name); + fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2)); fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME2\n"); rrddim_set_name(st, rd2, "DIM2NAME2"); - fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd2->id, rd2->name); + fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2)); BUFFER *buf = buffer_create(1); health_api_v1_chart_variables2json(st, buf); @@ -1449,7 +1449,7 @@ int unit_test(long delay, long shift) unsigned long c, dimensions = 0; RRDDIM *rd; - for(rd = st->dimensions ; rd ; rd = rd->next) dimensions++; + for(rd = st->dimensions; rd ; rd = rd->next) dimensions++; for(c = 0; c < 20 ;c++) { i += increment; @@ -1470,7 +1470,7 @@ int unit_test(long delay, long shift) } // prevent it from deleting the dimensions - for(rd = st->dimensions ; rd ; rd = rd->next) + for(rd = st->dimensions; rd ; rd = rd->next) rd->last_collected_time.tv_sec = st->last_collected_time.tv_sec; rrdset_done(st); @@ -1486,10 +1486,10 @@ int unit_test(long delay, long shift) for(c = 0 ; c < st->counter ; c++) { fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10); - for(rd = st->dimensions ; rd ; rd = rd->next) { + for(rd = st->dimensions; rd ; rd = rd->next) { sn = rd->db[c]; cn = unpack_storage_number(sn); - fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rd->id, cn, sn); + fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rrddim_id(rd), cn, sn); if(rd == rdabs) v = ( oincrement @@ -1909,14 +1909,14 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI if(!value_errors) fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT ", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n", - st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, expected, value); + rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now + k * update_every, expected, value); value_errors++; errors++; } if(end_time != time_now + k * update_every) { if(!time_errors) fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n", - st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, (unsigned long)time_retrieved); + rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now + k * update_every, (unsigned long)time_retrieved); time_errors++; errors++; } @@ -1957,7 +1957,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] NULL, NULL, NULL, 0, 0); if (!r) { - fprintf(stderr, " DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", st[i]->name, current_region); + fprintf(stderr, " DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", rrdset_name(st[i]), current_region); return ++errors; } else { assert(r->st == st[i]); @@ -1967,7 +1967,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] time_retrieved = r->t[c]; // for each dimension - for (j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) { + for (j = 0, d = r->st->dimensions; d && j < r->d ; ++j, d = d->next) { NETDATA_DOUBLE *cn = &r->v[ c * r->d ]; value = cn[j]; assert(rd[i][j] == d); @@ -1980,13 +1980,13 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] if(value_errors < 20) fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT ", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n", - st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value); + rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, expected, value); value_errors++; } if(time_retrieved != time_now) { if(time_errors < 20) fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n", - st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved); + rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, (unsigned long)time_retrieved); time_errors++; } } @@ -2090,7 +2090,7 @@ int test_dbengine(void) time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, RRDR_OPTION_NATURAL_POINTS, NULL, NULL, NULL, 0, 0); if (!r) { - fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name); + fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", rrdset_name(st[i])); ++errors; } else { long c; @@ -2103,7 +2103,7 @@ int test_dbengine(void) time_t time_retrieved = r->t[c]; // for each dimension - for(j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) { + for(j = 0, d = r->st->dimensions; d && j < r->d ; ++j, d = d->next) { NETDATA_DOUBLE *cn = &r->v[ c * r->d ]; NETDATA_DOUBLE value = cn[j]; assert(rd[i][j] == d); @@ -2116,13 +2116,13 @@ int test_dbengine(void) if(!value_errors) fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT ", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n", - st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value); + rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, expected, value); value_errors++; } if(time_retrieved != time_now) { if(!time_errors) fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n", - st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved); + rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, (unsigned long)time_retrieved); time_errors++; } } @@ -2350,7 +2350,7 @@ static void query_dbengine_chart(void *arg) if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT ", found data gap, ### E R R O R ###\n", - st->name, rd->name, (unsigned long) time_now, expected); + rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected); ++thread_info->errors; } break; @@ -2365,7 +2365,7 @@ static void query_dbengine_chart(void *arg) if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT ", found data gap, ### E R R O R ###\n", - st->name, rd->name, (unsigned long) time_now, expected); + rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected); ++thread_info->errors; } break; @@ -2378,7 +2378,7 @@ static void query_dbengine_chart(void *arg) if(!value_errors) fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT ", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n", - st->name, rd->name, (unsigned long) time_now, expected, value); + rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected, value); value_errors++; thread_info->errors++; } @@ -2388,7 +2388,7 @@ static void query_dbengine_chart(void *arg) if(!time_errors) fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n", - st->name, rd->name, (unsigned long) time_now, (unsigned long) time_retrieved); + rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, (unsigned long) time_retrieved); time_errors++; thread_info->errors++; } diff --git a/database/engine/metadata_log/metalogpluginsd.c b/database/engine/metadata_log/metalogpluginsd.c index a5301bc108..dcf2deb7d2 100755 --- a/database/engine/metadata_log/metalogpluginsd.c +++ b/database/engine/metadata_log/metalogpluginsd.c @@ -12,11 +12,11 @@ PARSER_RC metalog_pluginsd_host_action( { struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private; - RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0); + RRDHOST *host = rrdhost_find_by_guid(machine_guid); if (host) { if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) { error("Archived host '%s' has memory mode '%s', but the archived one is '%s'. Ignoring archived state.", - host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), + rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(RRD_MEMORY_MODE_DBENGINE)); ((PARSER_USER_OBJECT *) user)->host = NULL; /* Ignore objects if memory mode is not dbengine */ } diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index f4da294079..5acab14d3b 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -98,7 +98,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_insta pg_cache = &ctx->pg_cache; - rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid); + rrdeng_generate_legacy_uuid(rrddim_id(rd), (char *)rrdset_id(rd->rrdset), &legacy_uuid); if (host != localhost && is_storage_engine_shared((STORAGE_INSTANCE *)ctx)) is_multihost_child = 1; @@ -138,8 +138,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_insta uuid_copy(rd->metric_uuid, multihost_legacy_uuid); if (unlikely(need_to_store && !ctx->tier)) - (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor, - rd->algorithm); + (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); } struct rrdeng_metric_handle *mh = mallocz(sizeof(struct rrdeng_metric_handle)); diff --git a/database/ram/rrddim_mem.c b/database/ram/rrddim_mem.c index 3226d3c0de..139f997c93 100644 --- a/database/ram/rrddim_mem.c +++ b/database/ram/rrddim_mem.c @@ -91,7 +91,7 @@ static inline size_t rrddim_time2slot(RRDDIM *rd, time_t t) { } if(unlikely(ret >= entries)) { - error("INTERNAL ERROR: rrddim_time2slot() on %s returns values outside entries", rd->name); + error("INTERNAL ERROR: rrddim_time2slot() on %s returns values outside entries", rrddim_name(rd)); ret = entries - 1; } @@ -119,12 +119,12 @@ static inline time_t rrddim_slot2time(RRDDIM *rd, size_t slot) { ret = last_entry_t - (time_t)(update_every * (last_slot - slot)); if(unlikely(ret < first_entry_t)) { - error("INTERNAL ERROR: rrddim_slot2time() on %s returns time too far in the past", rd->name); + error("INTERNAL ERROR: rrddim_slot2time() on %s returns time too far in the past", rrddim_name(rd)); ret = first_entry_t; } if(unlikely(ret > last_entry_t)) { - error("INTERNAL ERROR: rrddim_slot2time() on %s returns time into the future", rd->name); + error("INTERNAL ERROR: rrddim_slot2time() on %s returns time into the future", rrddim_name(rd)); ret = last_entry_t; } @@ -206,7 +206,7 @@ int rrddim_query_is_finished(struct rrddim_query_handle *handle) { void rrddim_query_finalize(struct rrddim_query_handle *handle) { #ifdef NETDATA_INTERNAL_CHECKS if(!rrddim_query_is_finished(handle)) - error("QUERY: query for chart '%s' dimension '%s' has been stopped unfinished", handle->rd->rrdset->id, handle->rd->name); + error("QUERY: query for chart '%s' dimension '%s' has been stopped unfinished", rrdset_id(handle->rd->rrdset), rrddim_name(handle->rd)); #endif freez(handle->handle); } diff --git a/database/rrd.c b/database/rrd.c index f91039ea57..df364419ea 100644 --- a/database/rrd.c +++ b/database/rrd.c @@ -154,3 +154,15 @@ char *rrdset_cache_dir(RRDHOST *host, const char *id) { return ret; } +// ---------------------------------------------------------------------------- +// RRD - string management + +STRING *rrd_string_strdupz(const char *s) { + if(unlikely(!s || !*s)) return string_strdupz(s); + + char *tmp = strdupz(s); + json_fix_string(tmp); + STRING *ret = string_strdupz(tmp); + freez(tmp); + return ret; +} diff --git a/database/rrd.h b/database/rrd.h index f8919e83a5..36c4e5d58f 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -160,14 +160,10 @@ extern const char *rrd_algorithm_name(RRD_ALGORITHM algorithm); // RRD FAMILY struct rrdfamily { - avl_t avl; - - const char *family; - uint32_t hash_family; + STRING *family; + DICTIONARY *rrdvar_root_index; size_t use_count; - - avl_tree_lock rrdvar_root_index; }; typedef struct rrdfamily RRDFAMILY; @@ -190,6 +186,7 @@ typedef enum rrddim_flags { RRDDIM_FLAG_PENDING_FOREACH_ALARM = (1 << 5), // set when foreach alarm has not been initialized yet RRDDIM_FLAG_META_HIDDEN = (1 << 6), // Status of hidden option in the metadata database + RRDDIM_FLAG_INDEXED_ID = (1 << 7), } RRDDIM_FLAGS; #define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_SEQ_CST) & (flag)) @@ -244,27 +241,13 @@ extern bool exporting_labels_filter_callback(const char *name, const char *value // RRD DIMENSION - this is a metric struct rrddim { - // ------------------------------------------------------------------------ - // binary indexing structures - - avl_t avl; // the binary index - this has to be first member! - uuid_t metric_uuid; // global UUID for this metric (unique_across hosts) // ------------------------------------------------------------------------ // the dimension definition - const char *id; // the id of this dimension (for internal identification) - const char *name; // the name of this dimension (as presented to user) - // this is a pointer to the config structure - // since the config always has a higher priority - // (the user overwrites the name of the charts) - uint32_t hash; // a simple hash of the id, to speed up searching / indexing - // instead of strcmp() every item in the binary index - // we first compare the hashes - uint32_t hash_name; // a simple hash of the name - - + STRING *id; // the id of this dimension (for internal identification) + STRING *name; // the name of this dimension (as presented to user) RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension RRDDIM_FLAGS flags; // configuration flags for the dimension @@ -305,6 +288,8 @@ struct rrddim { NETDATA_DOUBLE stored_volume; // the sum of all stored values so far struct rrddim *next; // linking of dimensions within the same data set + struct rrddim *prev; // linking of dimensions within the same data set + struct rrdset *rrdset; RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension @@ -328,6 +313,9 @@ struct rrddim { storage_number *db; // the array of values }; +#define rrddim_id(rd) string2str((rd)->id) +#define rrddim_name(rd) string2str((rd) ->name) + // returns the RRDDIM cache filename, or NULL if it does not exist extern const char *rrddim_cache_filename(RRDDIM *rd); @@ -459,9 +447,6 @@ extern void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t n // ---------------------------------------------------------------------------- // volatile state per chart struct rrdset_volatile { - char *old_title; - char *old_units; - char *old_context; uuid_t hash_id; DICTIONARY *chart_labels; bool is_ar_chart; @@ -485,88 +470,75 @@ struct rrdset_volatile { // and may lead to missing information. typedef enum rrdset_flags { - RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another - // (the master data set should be the one that has the same family and is not detail) - RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart - RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete - RRDSET_FLAG_EXPORTING_SEND = 1 << 4, // if set, this chart should be sent to Prometheus web API and external databases - RRDSET_FLAG_EXPORTING_IGNORE = 1 << 5, // if set, this chart should not be sent to Prometheus web API and external databases - RRDSET_FLAG_UPSTREAM_SEND = 1 << 6, // if set, this chart should be sent upstream (streaming) - RRDSET_FLAG_UPSTREAM_IGNORE = 1 << 7, // if set, this chart should not be sent upstream (streaming) - RRDSET_FLAG_UPSTREAM_EXPOSED = 1 << 8, // if set, we have sent this chart definition to netdata parent (streaming) - RRDSET_FLAG_STORE_FIRST = 1 << 9, // if set, do not eliminate the first collection during interpolation - RRDSET_FLAG_HETEROGENEOUS = 1 << 10, // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers) - RRDSET_FLAG_HOMOGENEOUS_CHECK = 1 << 11, // if set, the chart should be checked to determine if the dimensions are homogeneous - RRDSET_FLAG_HIDDEN = 1 << 12, // if set, do not show this chart on the dashboard, but use it for exporting - RRDSET_FLAG_SYNC_CLOCK = 1 << 13, // if set, microseconds on next data collection will be ignored (the chart will be synced to now) - RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14, // this is marked by the collector/module when a chart has obsolete dimensions - // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at - // least rrdset_free_obsolete_time seconds ago. - RRDSET_FLAG_ARCHIVED = 1 << 15, - RRDSET_FLAG_ACLK = 1 << 16, - RRDSET_FLAG_PENDING_FOREACH_ALARMS = 1 << 17, // contains dims with uninitialized foreach alarms - RRDSET_FLAG_ANOMALY_DETECTION = 1 << 18 // flag to identify anomaly detection charts. + RRDSET_FLAG_DETAIL = (1 << 1), // if set, the data set should be considered as a detail of another + // (the master data set should be the one that has the same family and is not detail) + RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart + RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete + RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases + RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases + RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming) + RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming) + RRDSET_FLAG_UPSTREAM_EXPOSED = (1 << 8), // if set, we have sent this chart definition to netdata parent (streaming) + RRDSET_FLAG_STORE_FIRST = (1 << 9), // if set, do not eliminate the first collection during interpolation + RRDSET_FLAG_HETEROGENEOUS = (1 << 10), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers) + RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 11), // if set, the chart should be checked to determine if the dimensions are homogeneous + RRDSET_FLAG_HIDDEN = (1 << 12), // if set, do not show this chart on the dashboard, but use it for exporting + RRDSET_FLAG_SYNC_CLOCK = (1 << 13), // if set, microseconds on next data collection will be ignored (the chart will be synced to now) + RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 14), // this is marked by the collector/module when a chart has obsolete dimensions + // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at + // least rrdset_free_obsolete_time seconds ago. + RRDSET_FLAG_ARCHIVED = (1 << 15), + RRDSET_FLAG_ACLK = (1 << 16), + RRDSET_FLAG_PENDING_FOREACH_ALARMS = (1 << 17), // contains dims with uninitialized foreach alarms + RRDSET_FLAG_ANOMALY_DETECTION = (1 << 18), // flag to identify anomaly detection charts. + RRDSET_FLAG_INDEXED_ID = (1 << 19), + RRDSET_FLAG_INDEXED_NAME = (1 << 20), } RRDSET_FLAGS; #define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_SEQ_CST) & (flag)) #define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_SEQ_CST) -#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~flag, __ATOMIC_SEQ_CST) +#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~(flag), __ATOMIC_SEQ_CST) struct rrdset { - // ------------------------------------------------------------------------ - // binary indexing structures - - avl_t avl; // the index, with key the id - this has to be first! - avl_t avlname; // the index, with key the name - // ------------------------------------------------------------------------ // the set configuration - char id[RRD_ID_LENGTH_MAX + 1]; // id of the data set - - const char *name; // the name of this dimension (as presented to user) - // this is a pointer to the config structure - // since the config always has a higher priority - // (the user overwrites the name of the charts) - - char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) - char *family; // grouping sets under the same family - char *title; // title shown to user - char *units; // units of measurement - - char *context; // the template of this data set - uint32_t hash_context; // the hash of the chart's context + STRING *id; // the ID of the data set + STRING *name; // the name of this dimension (as presented to user) + STRING *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) + STRING *family; // grouping sets under the same family + STRING *title; // title shown to user + STRING *units; // units of measurement + STRING *context; // the template of this data set + STRING *plugin_name; // the name of the plugin that generated this + STRING *module_name; // the name of the plugin module that generated this RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to + RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset RRDSET_TYPE chart_type; // line, area, stacked + RRDSET_FLAGS flags; // configuration flags + RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances int update_every; // every how many seconds is this updated? + int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored + // netdata will interpolate values for gaps lower than this + long entries; // total number of entries in the data set long current_entry; // the entry that is currently being updated // it goes around in a round-robin fashion - RRDSET_FLAGS flags; // configuration flags - RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances - - int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored - // netdata will interpolate values for gaps lower than this - long priority; // the sorting priority of this chart // ------------------------------------------------------------------------ // members for temporary data we need for calculations - RRD_MEMORY_MODE rrd_memory_mode; // if set to 1, this is memory mapped - char *cache_dir; // the directory to store dimensions - netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list - size_t counter; // the number of times we added values to this database size_t counter_done; // the number of times rrdset_done() has been called @@ -576,19 +548,12 @@ struct rrdset { }; time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream - char *plugin_name; // the name of the plugin that generated this - char *module_name; // the name of the plugin module that generated this uuid_t *chart_uuid; // Store the global GUID for this chart // this object. struct rrdset_volatile *state; // volatile state that is not persistently stored size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine - uint32_t hash; // a simple hash on the id, to speed up searching - // we first compare hashes, and only if the hashes are equal we do string comparisons - - uint32_t hash_name; // a simple hash on the name - usec_t usec_since_last_update; // the time in microseconds since the last collection of data struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function) @@ -601,14 +566,15 @@ struct rrdset { RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to struct rrdset *next; // linking of rrdsets + struct rrdset *prev; // linking of rrdsets // ------------------------------------------------------------------------ // local variables - NETDATA_DOUBLE green; // green threshold for this chart - NETDATA_DOUBLE red; // red threshold for this chart + NETDATA_DOUBLE green; // green threshold for this chart + NETDATA_DOUBLE red; // red threshold for this chart - avl_tree_lock rrdvar_root_index; // RRDVAR index for this chart + DICTIONARY *rrdvar_root_index; // RRDVAR index for this chart RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs) RRDCALC *alarms; // RRDCALC linked list for this chart @@ -621,14 +587,27 @@ struct rrdset { // ------------------------------------------------------------------------ // the dimensions - avl_tree_lock dimensions_index; // the root of the dimensions index + DICTIONARY *rrddim_root_index; // the root of the dimensions index + + netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list RRDDIM *dimensions; // the actual data for every dimension }; +#define rrdset_plugin_name(st) string2str((st)->plugin_name) +#define rrdset_module_name(st) string2str((st)->module_name) +#define rrdset_units(st) string2str((st)->units) +#define rrdset_type(st) string2str((st)->type) +#define rrdset_family(st) string2str((st)->family) +#define rrdset_title(st) string2str((st)->title) +#define rrdset_context(st) string2str((st)->context) +#define rrdset_name(st) string2str((st)->name) +#define rrdset_id(st) string2str((st)->id) + #define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock)) #define rrdset_wrlock(st) netdata_rwlock_wrlock(&((st)->rrdset_rwlock)) #define rrdset_unlock(st) netdata_rwlock_unlock(&((st)->rrdset_rwlock)) +extern STRING *rrd_string_strdupz(const char *s); // ---------------------------------------------------------------------------- // these loop macros make sure the linked list is accessed with the right lock @@ -653,26 +632,29 @@ extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY // and may lead to missing information. typedef enum rrdhost_flags { - RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data) - RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts - RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan - RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases - RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases - RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet - RRDHOST_FLAG_MULTIHOST = (1 << 6), // Host belongs to localhost/megadb - RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms - RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8), - RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9), - RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates + RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data) + RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts + RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan + RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases + RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases + RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet + RRDHOST_FLAG_MULTIHOST = (1 << 6), // Host belongs to localhost/megadb + RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms + RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8), + RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9), + RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates + RRDHOST_FLAG_INDEXED_MACHINE_GUID = (1 << 11), // when set, we have indexed its machine guid + RRDHOST_FLAG_INDEXED_HOSTNAME = (1 << 12), // when set, we have indexed its hostname + RRDHOST_FLAG_STREAM_COLLECTED_METRICS = (1 << 13), // when set, rrdset_done() should push metrics to parent } RRDHOST_FLAGS; #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag)) #define rrdhost_flag_set(host, flag) __atomic_or_fetch(&((host)->flags), flag, __ATOMIC_SEQ_CST) -#define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~flag, __ATOMIC_SEQ_CST) +#define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~(flag), __ATOMIC_SEQ_CST) #ifdef NETDATA_INTERNAL_CHECKS #define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \ - debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, st->name, ##args); } while(0) + debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0) #else #define rrdset_debug(st, fmt, args...) debug_dummy() #endif @@ -690,34 +672,30 @@ struct alarm_entry { time_t duration; time_t non_clear_duration; - char *name; - uint32_t hash_name; + STRING *name; + STRING *chart; + STRING *chart_context; + STRING *family; - char *chart; - uint32_t hash_chart; - char *chart_context; + STRING *classification; + STRING *component; + STRING *type; - char *family; - - char *classification; - char *component; - char *type; - - char *exec; - char *recipient; + STRING *exec; + STRING *recipient; time_t exec_run_timestamp; int exec_code; uint64_t exec_spawn_serial; - char *source; - char *units; - char *info; + STRING *source; + STRING *units; + STRING *info; NETDATA_DOUBLE old_value; NETDATA_DOUBLE new_value; - char *old_value_string; - char *new_value_string; + STRING *old_value_string; + STRING *new_value_string; RRDCALC_STATUS old_status; RRDCALC_STATUS new_status; @@ -737,6 +715,20 @@ struct alarm_entry { struct alarm_entry *prev_in_progress; }; +#define ae_name(ae) string2str((ae)->name) +#define ae_chart_name(ae) string2str((ae)->chart) +#define ae_chart_context(ae) string2str((ae)->chart_context) +#define ae_family(ae) string2str((ae)->family) +#define ae_classification(ae) string2str((ae)->classification) +#define ae_component(ae) string2str((ae)->component) +#define ae_type(ae) string2str((ae)->type) +#define ae_exec(ae) string2str((ae)->exec) +#define ae_recipient(ae) string2str((ae)->recipient) +#define ae_source(ae) string2str((ae)->source) +#define ae_units(ae) string2str((ae)->units) +#define ae_info(ae) string2str((ae)->info) +#define ae_old_value_string(ae) string2str((ae)->old_value_string) +#define ae_new_value_string(ae) string2str((ae)->new_value_string) typedef struct alarm_log { uint32_t next_log_id; @@ -790,24 +782,20 @@ struct rrdhost_system_info { }; struct rrdhost { - avl_t avl; // the index of hosts + char machine_guid[GUID_LEN + 1]; // the unique ID of this host // ------------------------------------------------------------------------ // host information - char *hostname; // the hostname of this host - uint32_t hash_hostname; // the hostname hash + STRING *hostname; // the hostname of this host + STRING *registry_hostname; // the registry hostname for this host + STRING *os; // the O/S type of the host + STRING *tags; // tags for this host + STRING *timezone; // the timezone of the host + STRING *abbrev_timezone; // the abbriviated timezone of the host + STRING *program_name; // the program name that collects metrics for this host + STRING *program_version; // the program version that collects metrics for this host - char *registry_hostname; // the registry hostname for this host - - char machine_guid[GUID_LEN + 1]; // the unique ID of this host - uint32_t hash_machine_guid; // the hash of the unique ID - - const char *os; // the O/S type of the host - const char *tags; // tags for this host - const char *timezone; // the timezone of the host - - const char *abbrev_timezone; // the abbriviated timezone of the host int32_t utc_offset; // the offset in seconds from utc RRDHOST_FLAGS flags; // flags about this RRDHOST @@ -820,9 +808,6 @@ struct rrdhost { char *cache_dir; // the directory to save RRD cache files char *varlib_dir; // the directory to save health log - char *program_name; // the program name that collects metrics for this host - char *program_version; // the program version that collects metrics for this host - struct rrdhost_system_info *system_info; // information collected from the host environment // ------------------------------------------------------------------------ @@ -873,23 +858,21 @@ struct rrdhost { // health monitoring options unsigned int health_enabled; // 1 when this host has health enabled - time_t health_delay_up_to; // a timestamp to delay alarms processing up to - char *health_default_exec; // the full path of the alarms notifications program - char *health_default_recipient; // the default recipient for all alarms - char *health_log_filename; // the alarms event log filename - size_t health_log_entries_written; // the number of alarm events written to the alarms event log - FILE *health_log_fp; // the FILE pointer to the open alarms event log file - uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications - uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications + time_t health_delay_up_to; // a timestamp to delay alarms processing up to + STRING *health_default_exec; // the full path of the alarms notifications program + STRING *health_default_recipient; // the default recipient for all alarms + char *health_log_filename; // the alarms event log filename + size_t health_log_entries_written; // the number of alarm events written to the alarms event log + FILE *health_log_fp; // the FILE pointer to the open alarms event log file + uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications + uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications // all RRDCALCs are primarily allocated and linked here // RRDCALCs may be linked to charts at any point // (charts may or may not exist when these are loaded) - RRDCALC *alarms; + RRDCALC *host_alarms; RRDCALC *alarms_with_foreach; - avl_tree_lock alarms_idx_health_log; - avl_tree_lock alarms_idx_name; ALARM_LOG health_log; // alarms historical events (event log) uint32_t health_last_processed_id; // the last processed health id from the log @@ -899,9 +882,7 @@ struct rrdhost { // templates of alarms // these are used to create alarms when charts // are created or renamed, that match them - RRDCALCTEMPLATE *templates; - RRDCALCTEMPLATE *alarms_template_with_foreach; - + RRDCALCTEMPLATE *alarms_templates; // ------------------------------------------------------------------------ // the charts of the host @@ -927,11 +908,11 @@ struct rrdhost { // ------------------------------------------------------------------------ // indexes - avl_tree_lock rrdset_root_index; // the host's charts index (by id) - avl_tree_lock rrdset_root_index_name; // the host's charts index (by name) + DICTIONARY *rrdset_root_index; // the host's charts index (by id) + DICTIONARY *rrdset_root_index_name; // the host's charts index (by name) - avl_tree_lock rrdfamily_root_index; // the host's chart families index - avl_tree_lock rrdvar_root_index; // the host's chart variables index + DICTIONARY *rrdfamily_root_index; // the host's chart families index + DICTIONARY *rrdvar_root_index; // the host's chart variables index STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers @@ -950,9 +931,19 @@ struct rrdhost { aclk_rrdhost_state aclk_state; struct rrdhost *next; + struct rrdhost *prev; }; extern RRDHOST *localhost; +#define rrdhost_hostname(host) string2str((host)->hostname) +#define rrdhost_registry_hostname(host) string2str((host)->registry_hostname) +#define rrdhost_os(host) string2str((host)->os) +#define rrdhost_tags(host) string2str((host)->tags) +#define rrdhost_timezone(host) string2str((host)->timezone) +#define rrdhost_abbrev_timezone(host) string2str((host)->abbrev_timezone) +#define rrdhost_program_name(host) string2str((host)->program_name) +#define rrdhost_program_version(host) string2str((host)->program_version) + #define rrdhost_rdlock(host) netdata_rwlock_rdlock(&((host)->rrdhost_rwlock)) #define rrdhost_wrlock(host) netdata_rwlock_wrlock(&((host)->rrdhost_rwlock)) #define rrdhost_unlock(host) netdata_rwlock_unlock(&((host)->rrdhost_rwlock)) @@ -960,6 +951,8 @@ extern RRDHOST *localhost; #define rrdhost_aclk_state_lock(host) netdata_mutex_lock(&((host)->aclk_state_lock)) #define rrdhost_aclk_state_unlock(host) netdata_mutex_unlock(&((host)->aclk_state_lock)) +extern long rrdhost_hosts_available(void); + // ---------------------------------------------------------------------------- // these loop macros make sure the linked list is accessed with the right lock @@ -990,8 +983,8 @@ extern time_t rrdhost_free_orphan_time; extern int rrd_init(char *hostname, struct rrdhost_system_info *system_info); -extern RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash); -extern RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash); +extern RRDHOST *rrdhost_find_by_hostname(const char *hostname); +extern RRDHOST *rrdhost_find_by_guid(const char *guid); extern RRDHOST *rrdhost_find_or_create( const char *hostname @@ -1237,11 +1230,18 @@ time_t rrdhost_last_entry_t(RRDHOST *h); // RRD DIMENSION functions extern void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host); -extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, - collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode);//, - //int is_archived, uuid_t *dim_uuid); -#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, \ - algorithm, (st)->rrd_memory_mode)//, 0, NULL) + +extern RRDDIM *rrddim_add_custom(RRDSET *st + , const char *id + , const char *name + , collected_number multiplier + , collected_number divisor + , RRD_ALGORITHM algorithm + , RRD_MEMORY_MODE memory_mode + ); + +#define rrddim_add(st, id, name, multiplier, divisor, algorithm) \ + rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode) extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name); extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm); @@ -1249,15 +1249,7 @@ extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor); extern RRDDIM *rrddim_find(RRDSET *st, const char *id); -/* This will not return dimensions that are archived */ -static inline RRDDIM *rrddim_find_active(RRDSET *st, const char *id) -{ - RRDDIM *rd = rrddim_find(st, id); - if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))) - return NULL; - return rd; -} - +extern RRDDIM *rrddim_find_active(RRDSET *st, const char *id); extern int rrddim_hide(RRDSET *st, const char *id); extern int rrddim_unhide(RRDSET *st, const char *id); @@ -1275,33 +1267,20 @@ extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries); // ---------------------------------------------------------------------------- // Miscellaneous functions -extern int alarm_compare_id(void *a, void *b); -extern int alarm_compare_name(void *a, void *b); +extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length); // ---------------------------------------------------------------------------- // RRD internal functions #ifdef NETDATA_RRD_INTERNALS -extern avl_tree_lock rrdhost_root_index; - -extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length); extern char *rrdset_cache_dir(RRDHOST *host, const char *id); extern void rrddim_free(RRDSET *st, RRDDIM *rd); -extern int rrddim_compare(void* a, void* b); -extern int rrdset_compare(void* a, void* b); -extern int rrdset_compare_name(void* a, void* b); -extern int rrdfamily_compare(void *a, void *b); - extern RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id); extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc); -#define rrdset_index_add(host, st) (RRDSET *)avl_insert_lock(&((host)->rrdset_root_index), (avl_t *)(st)) -#define rrdset_index_del(host, st) (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index), (avl_t *)(st)) -extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st); - extern void rrdset_free(RRDSET *st); extern void rrdset_reset(RRDSET *st); extern void rrdset_save(RRDSET *st); @@ -1318,8 +1297,8 @@ extern RRDHOST *rrdhost_create( #endif /* NETDATA_RRD_INTERNALS */ extern void set_host_properties( - RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *hostname, const char *registry_hostname, - const char *guid, const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset, + RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname, + const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *program_name, const char *program_version); extern int get_tier_grouping(int tier); diff --git a/database/rrdcalc.c b/database/rrdcalc.c index 39fb09681a..808e74952a 100644 --- a/database/rrdcalc.c +++ b/database/rrdcalc.c @@ -35,9 +35,8 @@ inline const char *rrdcalc_status2string(RRDCALC_STATUS status) { } } -char *rrdcalc_replace_variables(const char *line, RRDCALC *rc) -{ - if (!line) +static STRING *rrdcalc_replace_variables(const char *line, RRDCALC *rc) { + if (!line || !*line) return NULL; size_t pos = 0; @@ -46,24 +45,28 @@ char *rrdcalc_replace_variables(const char *line, RRDCALC *rc) char *m, *lbl_value = NULL; while ((m = strchr(temp + pos, '$'))) { - int i=0; + int i = 0; char *e = m; while (*e) { - if (*e == ' ' || i == RRDCALC_VAR_MAX - 1) { + + if (*e == ' ' || i == RRDCALC_VAR_MAX - 1) break; - } else - var[i]=*e; + var[i] = *e; + e++; i++; } - var[i]='\0'; + + var[i] = '\0'; pos = m - temp + 1; + if (!strcmp(var, RRDCALC_VAR_FAMILY)) { - char *buf = find_and_replace(temp, var, (rc->rrdset && rc->rrdset->family) ? rc->rrdset->family : "", m); + char *buf = find_and_replace(temp, var, (rc->rrdset && rc->rrdset->family) ? rrdset_family(rc->rrdset) : "", m); freez(temp); temp = buf; - } else if (!strncmp(var, RRDCALC_VAR_LABEL, RRDCALC_VAR_LABEL_LEN)) { + } + else if (!strncmp(var, RRDCALC_VAR_LABEL, RRDCALC_VAR_LABEL_LEN)) { if(likely(rc->rrdset && rc->rrdset->state && rc->rrdset->state->chart_labels)) { rrdlabels_get_value_to_char_or_null(rc->rrdset->state->chart_labels, &lbl_value, var+RRDCALC_VAR_LABEL_LEN); if (lbl_value) { @@ -76,17 +79,20 @@ char *rrdcalc_replace_variables(const char *line, RRDCALC *rc) } } - return temp; + STRING *ret = string_strdupz(temp); + freez(temp); + + return ret; } void rrdcalc_update_rrdlabels(RRDSET *st) { RRDCALC *rc; - for( rc = st->alarms; rc ; rc = rc->rrdset_next ) { + foreach_rrdcalc_in_rrdset(st, rc) { if (rc->original_info) { if (rc->info) - freez(rc->info); + string_freez(rc->info); - rc->info = rrdcalc_replace_variables(rc->original_info, rc); + rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc); } } } @@ -94,58 +100,57 @@ void rrdcalc_update_rrdlabels(RRDSET *st) { static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { RRDHOST *host = st->rrdhost; - debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname); + debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host)); rc->last_status_change = now_realtime_sec(); rc->rrdset = st; - rc->rrdset_next = st->alarms; - rc->rrdset_prev = NULL; - - if(rc->rrdset_next) - rc->rrdset_next->rrdset_prev = rc; - - st->alarms = rc; + DOUBLE_LINKED_LIST_PREPEND_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next); if(rc->update_every < rc->rrdset->update_every) { - error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rc->rrdset->id, rc->name, rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every); + error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every); rc->update_every = rc->rrdset->update_every; } if(!isnan(rc->green) && isnan(st->green)) { debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " NETDATA_DOUBLE_FORMAT_AUTO - " to " NETDATA_DOUBLE_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green); + " to " NETDATA_DOUBLE_FORMAT_AUTO ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->green, rc->green); st->green = rc->green; } if(!isnan(rc->red) && isnan(st->red)) { debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " NETDATA_DOUBLE_FORMAT_AUTO " to " NETDATA_DOUBLE_FORMAT_AUTO - ".", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red); + ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->red, rc->red); st->red = rc->red; } - rc->local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value); - rc->family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value); + rc->local = rrdvar_create_and_index("local", st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value); + rc->family = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value); char fullname[RRDVAR_MAX_LENGTH + 1]; - snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name); - rc->hostid = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value); + snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), rrdcalc_name(rc)); + STRING *fullname_string = string_strdupz(fullname); + rc->hostid = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value); - snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name); - rc->hostname = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value); + snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), rrdcalc_name(rc)); + rc->hostname = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value); + + string_freez(fullname_string); if(rc->hostid && !rc->hostname) rc->hostid->options |= RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR; - if(!rc->units) rc->units = strdupz(st->units); + if(!rc->units) rc->units = string_dup(st->units); if (rc->original_info) { if (rc->info) - freez(rc->info); - rc->info = rrdcalc_replace_variables(rc->original_info, rc); + string_freez(rc->info); + + rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc); } time_t now = now_realtime_sec(); + ALARM_ENTRY *ae = health_create_alarm_entry( host, rc->id, @@ -170,19 +175,20 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { rc->units, rc->info, 0, - 0); + rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0); + health_alarm_log(host, ae); } -static int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) { - if((rc->hash_chart != st->hash || strcmp(rc->chart, st->id) != 0) && - (rc->hash_chart != st->hash_name || strcmp(rc->chart, st->name) != 0)) +static inline int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) { + if ( (rc->chart != st->id) + && (rc->chart != st->name)) return 0; - if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, st->module_name)) + if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, rrdset_module_name(st))) return 0; - if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, st->plugin_name)) + if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, rrdset_plugin_name(st))) return 0; if (st->rrdhost->host_labels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->host_labels, rc->host_labels_pattern, '=')) @@ -197,7 +203,7 @@ inline void rrdsetcalc_link_matching(RRDSET *st) { // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id); RRDCALC *rc; - for(rc = host->alarms; rc ; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if(unlikely(rc->rrdset)) continue; @@ -211,14 +217,15 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { RRDSET *st = rc->rrdset; if(!st) { - debug(D_HEALTH, "Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name); - error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name); + debug(D_HEALTH, "Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rrdcalc_chart_name(rc), rrdcalc_name(rc)); + error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return; } RRDHOST *host = st->rrdhost; time_t now = now_realtime_sec(); + ALARM_ENTRY *ae = health_create_alarm_entry( host, rc->id, @@ -244,32 +251,24 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { rc->info, 0, 0); + health_alarm_log(host, ae); - debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname); + debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host)); // unlink it - if(rc->rrdset_prev) - rc->rrdset_prev->rrdset_next = rc->rrdset_next; + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next); - if(rc->rrdset_next) - rc->rrdset_next->rrdset_prev = rc->rrdset_prev; - - if(st->alarms == rc) - st->alarms = rc->rrdset_next; - - rc->rrdset_prev = rc->rrdset_next = NULL; - - rrdvar_free(host, &st->rrdvar_root_index, rc->local); + rrdvar_free(host, st->rrdvar_root_index, rc->local); rc->local = NULL; - rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rc->family); + rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rc->family); rc->family = NULL; - rrdvar_free(host, &host->rrdvar_root_index, rc->hostid); + rrdvar_free(host, host->rrdvar_root_index, rc->hostid); rc->hostid = NULL; - rrdvar_free(host, &host->rrdvar_root_index, rc->hostname); + rrdvar_free(host, host->rrdvar_root_index, rc->hostname); rc->hostname = NULL; rc->rrdset = NULL; @@ -280,18 +279,21 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { } RRDCALC *rrdcalc_find(RRDSET *st, const char *name) { - RRDCALC *rc; - uint32_t hash = simple_hash(name); + RRDCALC *rc = NULL; - for( rc = st->alarms; rc ; rc = rc->rrdset_next ) { - if(unlikely(rc->hash == hash && !strcmp(rc->name, name))) - return rc; + STRING *name_string = string_strdupz(name); + + foreach_rrdcalc_in_rrdset(st, rc) { + if(unlikely(rc->name == name_string)) + break; } - return NULL; + string_freez(name_string); + + return rc; } -inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name) { +inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name) { RRDCALC *rc; if(unlikely(!chart)) { @@ -299,36 +301,39 @@ inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, ui return 1; } - if(unlikely(!hash_chart)) hash_chart = simple_hash(chart); - if(unlikely(!hash_name)) hash_name = simple_hash(name); + STRING *name_string = string_strdupz(name); + STRING *chart_string = string_strdupz(chart); + int ret = 0; // make sure it does not already exist - for(rc = host->alarms; rc ; rc = rc->next) { - if (unlikely(rc->chart && rc->hash == hash_name && rc->hash_chart == hash_chart && !strcmp(name, rc->name) && !strcmp(chart, rc->chart))) { - debug(D_HEALTH, "Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname); - info("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname); - return 1; + foreach_rrdcalc_in_rrdhost(host, rc) { + if (unlikely(rc->chart == chart_string && rc->name == name_string)) { + debug(D_HEALTH, "Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host)); + info("Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host)); + ret = 1; + break; } } - return 0; + string_freez(name_string); + string_freez(chart_string); + + return ret; } -inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id) { - if(chart && name) { - uint32_t hash_chart = simple_hash(chart); - uint32_t hash_name = simple_hash(name); - - // re-use old IDs, by looking them up in the alarm log - ALARM_ENTRY *ae; - for(ae = host->health_log.alarms; ae ;ae = ae->next) { - if(unlikely(ae->hash_name == hash_name && ae->hash_chart == hash_chart && !strcmp(name, ae->name) && !strcmp(chart, ae->chart))) { - if(next_event_id) *next_event_id = ae->alarm_event_id + 1; - return ae->alarm_id; - } +inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id) { + // re-use old IDs, by looking them up in the alarm log + ALARM_ENTRY *ae = NULL; + for(ae = host->health_log.alarms; ae ;ae = ae->next) { + if(unlikely(name == ae->name && chart == ae->chart)) { + if(next_event_id) *next_event_id = ae->alarm_event_id + 1; + break; } } + if(ae) + return ae->alarm_id; + if (unlikely(!host->health_log.next_alarm_id)) host->health_log.next_alarm_id = (uint32_t)now_realtime_sec(); @@ -347,7 +352,7 @@ inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const ch * * @return It returns the new name on success and the old otherwise */ -char *alarm_name_with_dim(char *name, size_t namelen, const char *dim, size_t dimlen) { +char *alarm_name_with_dim(const char *name, size_t namelen, const char *dim, size_t dimlen) { char *newname,*move; newname = mallocz(namelen + dimlen + 2); @@ -407,15 +412,7 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) { if(!rc->foreachdim) { // link it to the host alarms list - if(likely(host->alarms)) { - // append it - RRDCALC *t; - for(t = host->alarms; t && t->next ; t = t->next) ; - t->next = rc; - } - else { - host->alarms = rc; - } + DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->host_alarms, rc, prev, next); // link it to its chart RRDSET *st; @@ -425,43 +422,33 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) { break; } } - } else { - //link it case there is a foreach - if(likely(host->alarms_with_foreach)) { - // append it - RRDCALC *t; - for(t = host->alarms_with_foreach; t && t->next ; t = t->next) ; - t->next = rc; - } - else { - host->alarms_with_foreach = rc; - } - - //I am not linking this alarm direct to the host here, this will be done when the children is created + } + else { + DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->alarms_with_foreach, rc, prev, next); + // We are not linking this alarm directly to the host here + // It will eventually be done if it matches the dimensions } } inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) { - debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rt->name); + debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rrdcalctemplate_name(rt)); - if(rrdcalc_exists(host, chart, rt->name, 0, 0)) + if(rrdcalc_exists(host, chart, rrdcalctemplate_name(rt))) return NULL; RRDCALC *rc = callocz(1, sizeof(RRDCALC)); rc->next_event_id = 1; - rc->name = strdupz(rt->name); - rc->hash = simple_hash(rc->name); - rc->chart = strdupz(chart); - rc->hash_chart = simple_hash(rc->chart); + rc->name = string_dup(rt->name); + rc->chart = string_strdupz(chart); uuid_copy(rc->config_hash_id, rt->config_hash_id); rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id); - if(rt->dimensions) rc->dimensions = strdupz(rt->dimensions); - if(rt->foreachdim) { - rc->foreachdim = strdupz(rt->foreachdim); - rc->spdim = health_pattern_from_foreach(rc->foreachdim); - } + rc->dimensions = string_dup(rt->dimensions); + rc->foreachdim = string_dup(rt->foreachdim); + if(rt->foreachdim) + rc->spdim = health_pattern_from_foreach(rrdcalc_foreachdim(rc)); + rc->foreachcounter = rt->foreachcounter; rc->green = rt->green; @@ -485,55 +472,53 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, rc->update_every = rt->update_every; rc->options = rt->options; - if(rt->exec) rc->exec = strdupz(rt->exec); - if(rt->recipient) rc->recipient = strdupz(rt->recipient); - if(rt->source) rc->source = strdupz(rt->source); - if(rt->units) rc->units = strdupz(rt->units); - if(rt->info) { - rc->info = strdupz(rt->info); - rc->original_info = strdupz(rt->info); - } + rc->exec = string_dup(rt->exec); + rc->recipient = string_dup(rt->recipient); + rc->source = string_dup(rt->source); + rc->units = string_dup(rt->units); + rc->info = string_dup(rt->info); + rc->original_info = string_dup(rt->info); - if (rt->classification) rc->classification = strdupz(rt->classification); - if (rt->component) rc->component = strdupz(rt->component); - if (rt->type) rc->type = strdupz(rt->type); + rc->classification = string_dup(rt->classification); + rc->component = string_dup(rt->component); + rc->type = string_dup(rt->type); if(rt->calculation) { rc->calculation = expression_parse(rt->calculation->source, NULL, NULL); if(!rc->calculation) - error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, rt->name, rt->calculation->source); + error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, rrdcalctemplate_name(rt), rt->calculation->source); } if(rt->warning) { rc->warning = expression_parse(rt->warning->source, NULL, NULL); if(!rc->warning) - error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, rt->name, rt->warning->source); + error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, rrdcalctemplate_name(rt), rt->warning->source); } if(rt->critical) { rc->critical = expression_parse(rt->critical->source, NULL, NULL); if(!rc->critical) - error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source); + error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rrdcalctemplate_name(rt), rt->critical->source); } debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO ", red " NETDATA_DOUBLE_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", - (rc->chart)?rc->chart:"NOCHART", - rc->name, - (rc->exec)?rc->exec:"DEFAULT", - (rc->recipient)?rc->recipient:"DEFAULT", + rrdcalc_chart_name(rc), + rrdcalc_name(rc), + (rc->exec)?rrdcalc_exec(rc):"DEFAULT", + (rc->recipient)?rrdcalc_recipient(rc):"DEFAULT", rc->green, rc->red, (int)rc->group, rc->after, rc->before, rc->options, - (rc->dimensions)?rc->dimensions:"NONE", - (rc->foreachdim)?rc->foreachdim:"NONE", + (rc->dimensions)?rrdcalc_dimensions(rc):"NONE", + (rc->foreachdim)?rrdcalc_foreachdim(rc):"NONE", rc->update_every, (rc->calculation)?rc->calculation->parsed_as:"NONE", (rc->warning)?rc->warning->parsed_as:"NONE", (rc->critical)?rc->critical->parsed_as:"NONE", - rc->source, + rrdcalc_source(rc), rc->delay_up_duration, rc->delay_down_duration, rc->delay_max_duration, @@ -543,13 +528,6 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, ); rrdcalc_add_to_host(host, rc); - if(!rt->foreachdim) { - RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl_t *)rc); - if (rdcmp != rc) { - error("Cannot insert the alarm index ID %s",rc->name); - } - } - return rc; } @@ -570,14 +548,12 @@ inline RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const ch RRDCALC *newrc = callocz(1, sizeof(RRDCALC)); newrc->next_event_id = 1; - newrc->id = rrdcalc_get_unique_id(host, rc->chart, name, &rc->next_event_id); - newrc->name = (char *)name; - newrc->hash = simple_hash(newrc->name); - newrc->chart = strdupz(rc->chart); - newrc->hash_chart = simple_hash(rc->chart); + newrc->name = string_strdupz(name); + newrc->chart = string_dup(rc->chart); + newrc->id = rrdcalc_get_unique_id(host, newrc->chart, newrc->name, &rc->next_event_id); uuid_copy(newrc->config_hash_id, *((uuid_t *) &rc->config_hash_id)); - newrc->dimensions = strdupz(dimension); + newrc->dimensions = string_strdupz(dimension); newrc->foreachdim = NULL; rc->foreachcounter++; newrc->foreachcounter = rc->foreachcounter; @@ -603,33 +579,33 @@ inline RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const ch newrc->update_every = rc->update_every; newrc->options = rc->options; - if(rc->exec) newrc->exec = strdupz(rc->exec); - if(rc->recipient) newrc->recipient = strdupz(rc->recipient); - if(rc->source) newrc->source = strdupz(rc->source); - if(rc->units) newrc->units = strdupz(rc->units); - if(rc->info) newrc->info = strdupz(rc->info); - if(rc->original_info) newrc->original_info = strdupz(rc->original_info); + newrc->exec = string_dup(rc->exec); + newrc->recipient = string_dup(rc->recipient); + newrc->source = string_dup(rc->source); + newrc->units = string_dup(rc->units); + newrc->info = string_dup(rc->info); + newrc->original_info = string_dup(rc->original_info); - if (rc->classification) newrc->classification = strdupz(rc->classification); - if (rc->component) newrc->component = strdupz(rc->component); - if (rc->type) newrc->type = strdupz(rc->type); + newrc->classification = string_dup(rc->classification); + newrc->component = string_dup(rc->component); + newrc->type = string_dup(rc->type); if(rc->calculation) { newrc->calculation = expression_parse(rc->calculation->source, NULL, NULL); if(!newrc->calculation) - error("Health alarm '%s.%s': failed to parse calculation expression '%s'", rc->chart, rc->name, rc->calculation->source); + error("Health alarm '%s.%s': failed to parse calculation expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->calculation->source); } if(rc->warning) { newrc->warning = expression_parse(rc->warning->source, NULL, NULL); if(!newrc->warning) - error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", rc->chart, rc->name, rc->warning->source); + error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->warning->source); } if(rc->critical) { newrc->critical = expression_parse(rc->critical->source, NULL, NULL); if(!newrc->critical) - error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", rc->chart, rc->name, rc->critical->source); + error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->critical->source); } return newrc; @@ -642,86 +618,48 @@ void rrdcalc_free(RRDCALC *rc) { expression_free(rc->warning); expression_free(rc->critical); - freez(rc->name); - freez(rc->chart); - freez(rc->family); - freez(rc->dimensions); - freez(rc->foreachdim); - freez(rc->exec); - freez(rc->recipient); - freez(rc->source); - freez(rc->units); - freez(rc->info); - freez(rc->original_info); - freez(rc->classification); - freez(rc->component); - freez(rc->type); + string_freez(rc->name); + string_freez(rc->chart); + string_freez(rc->dimensions); + string_freez(rc->foreachdim); + string_freez(rc->exec); + string_freez(rc->recipient); + string_freez(rc->source); + string_freez(rc->units); + string_freez(rc->info); + string_freez(rc->original_info); + string_freez(rc->classification); + string_freez(rc->component); + string_freez(rc->type); + string_freez(rc->host_labels); + string_freez(rc->module_match); + string_freez(rc->plugin_match); + simple_pattern_free(rc->spdim); - freez(rc->host_labels); simple_pattern_free(rc->host_labels_pattern); - freez(rc->module_match); simple_pattern_free(rc->module_pattern); - freez(rc->plugin_match); simple_pattern_free(rc->plugin_pattern); + + freez(rc->family); freez(rc); } void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) { if(unlikely(!rc)) return; - debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname); + debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdhost_hostname(host)); // unlink it from RRDSET if(rc->rrdset) rrdsetcalc_unlink(rc); // unlink it from RRDHOST - if(unlikely(rc == host->alarms)) - host->alarms = rc->next; - else { - RRDCALC *t; - for(t = host->alarms; t && t->next != rc; t = t->next) ; - if(t) { - t->next = rc->next; - rc->next = NULL; - } - else - error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname); - } - - RRDCALC *rdcmp = (RRDCALC *) avl_search_lock(&(host)->alarms_idx_health_log, (avl_t *)rc); - if (rdcmp) { - rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_health_log, (avl_t *)rc); - if (!rdcmp) { - error("Cannot remove the health alarm index from health_log"); - } - } - - rdcmp = (RRDCALC *) avl_search_lock(&(host)->alarms_idx_name, (avl_t *)rc); - if (rdcmp) { - rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_name, (avl_t *)rc); - if (!rdcmp) { - error("Cannot remove the health alarm index from idx_name"); - } - } + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->host_alarms, rc, prev, next); rrdcalc_free(rc); } void rrdcalc_foreach_unlink_and_free(RRDHOST *host, RRDCALC *rc) { - - if(unlikely(rc == host->alarms_with_foreach)) - host->alarms_with_foreach = rc->next; - else { - RRDCALC *t; - for(t = host->alarms_with_foreach; t && t->next != rc; t = t->next) ; - if(t) { - t->next = rc->next; - rc->next = NULL; - } - else - error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname); - } - + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->alarms_with_foreach, rc, prev, next); rrdcalc_free(rc); } @@ -736,11 +674,11 @@ static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) { if(!rrdlabels_match_simple_pattern_parsed(host->host_labels, rc->host_labels_pattern, '=')) { info("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'", - rc->name, - host->hostname, - rc->host_labels); + rrdcalc_name(rc), + rrdhost_hostname(host), + rrdcalc_host_labels(rc)); - if(host->alarms == alarms) + if(host->host_alarms == alarms) rrdcalc_unlink_and_free(host, rc); else rrdcalc_foreach_unlink_and_free(host, rc); @@ -750,7 +688,7 @@ static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) { } void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host) { - rrdcalc_labels_unlink_alarm_loop(host, host->alarms); + rrdcalc_labels_unlink_alarm_loop(host, host->host_alarms); rrdcalc_labels_unlink_alarm_loop(host, host->alarms_with_foreach); } @@ -773,59 +711,3 @@ void rrdcalc_labels_unlink() { rrd_unlock(); } - -// ---------------------------------------------------------------------------- -// Alarm - - -/** - * Alarm is repeating - * - * Is this alarm repeating ? - * - * @param host The structure that has the binary tree - * @param alarm_id the id of the alarm to search - * - * @return It returns 1 case it is repeating and 0 otherwise - */ -int alarm_isrepeating(RRDHOST *host, uint32_t alarm_id) { - RRDCALC findme; - findme.id = alarm_id; - RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_health_log, (avl_t *)&findme); - if (!rc) { - return 0; - } - return rrdcalc_isrepeating(rc); -} - -/** - * Entry is repeating - * - * Check whether the id of alarm entry is yet present in the host structure - * - * @param host The structure that has the binary tree - * @param ae the alarm entry - * - * @return It returns 1 case it is repeating and 0 otherwise - */ -int alarm_entry_isrepeating(RRDHOST *host, ALARM_ENTRY *ae) { - return alarm_isrepeating(host, ae->alarm_id); -} - -/** - * Max last repeat - * - * Check the maximum last_repeat for the alarms associated a host - * - * @param host The structure that has the binary tree - * - * @return It returns 1 case it is repeating and 0 otherwise - */ -RRDCALC *alarm_max_last_repeat(RRDHOST *host, char *alarm_name,uint32_t hash) { - RRDCALC findme; - findme.name = alarm_name; - findme.hash = hash; - RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_name, (avl_t *)&findme); - - return rc; -} diff --git a/database/rrdcalc.h b/database/rrdcalc.h index 66daf844fe..5d356b9d69 100644 --- a/database/rrdcalc.h +++ b/database/rrdcalc.h @@ -33,33 +33,32 @@ struct rrdcalc { avl_t avl; // the index, with key the id - this has to be first! + uint32_t id; // the unique id of this alarm uint32_t next_event_id; // the next event id that will be used for this alarm - char *name; // the name of this alarm - uint32_t hash; // the hash of the alarm name uuid_t config_hash_id; // a predictable hash_id based on specific alert configuration - char *exec; // the command to execute when this alarm switches state - char *recipient; // the recipient of the alarm (the first parameter to exec) + STRING *name; // the name of this alarm + STRING *chart; // the chart id this should be linked to - char *classification; // the class that this alarm belongs - char *component; // the component that this alarm refers to - char *type; // type of the alarm + STRING *exec; // the command to execute when this alarm switches state + STRING *recipient; // the recipient of the alarm (the first parameter to exec) - char *chart; // the chart id this should be linked to - uint32_t hash_chart; + STRING *classification; // the class that this alarm belongs + STRING *component; // the component that this alarm refers to + STRING *type; // type of the alarm - char *plugin_match; //the plugin name that should be linked to + STRING *plugin_match; // the plugin name that should be linked to SIMPLE_PATTERN *plugin_pattern; - char *module_match; //the module name that should be linked to + STRING *module_match; // the module name that should be linked to SIMPLE_PATTERN *module_pattern; - char *source; // the source of this alarm - char *units; // the units of the alarm - char *original_info; // the original info field before any variable replacement - char *info; // a short description of the alarm + STRING *source; // the source of this alarm + STRING *units; // the units of the alarm + STRING *original_info; // the original info field before any variable replacement + STRING *info; // a short description of the alarm int update_every; // update frequency for the alarm @@ -70,8 +69,8 @@ struct rrdcalc { // ------------------------------------------------------------------------ // database lookup settings - char *dimensions; // the chart dimensions - char *foreachdim; // the group of dimensions that the `foreach` will be applied. + STRING *dimensions; // the chart dimensions + STRING *foreachdim; // the group of dimensions that the `foreach` will be applied. SIMPLE_PATTERN *spdim; // used if and only if there is a simple pattern for the chart. int foreachcounter; // the number of alarms created with foreachdim, this also works as an id of the // children @@ -99,29 +98,29 @@ struct rrdcalc { // ------------------------------------------------------------------------ // notification repeat settings - uint32_t warn_repeat_every; // interval between repeating warning notifications - uint32_t crit_repeat_every; // interval between repeating critical notifications + uint32_t warn_repeat_every; // interval between repeating warning notifications + uint32_t crit_repeat_every; // interval between repeating critical notifications // ------------------------------------------------------------------------ // Labels settings - char *host_labels; // the label read from an alarm file + STRING *host_labels; // the label read from an alarm file SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels // ------------------------------------------------------------------------ // runtime information - RRDCALC_STATUS old_status; // the old status of the alarm + RRDCALC_STATUS old_status; // the old status of the alarm RRDCALC_STATUS status; // the current status of the alarm - NETDATA_DOUBLE value; // the current value of the alarm - NETDATA_DOUBLE old_value; // the previous value of the alarm + NETDATA_DOUBLE value; // the current value of the alarm + NETDATA_DOUBLE old_value; // the previous value of the alarm uint32_t rrdcalc_flags; // check RRDCALC_FLAG_* time_t last_updated; // the last update timestamp of the alarm time_t next_update; // the next update timestamp of the alarm time_t last_status_change; // the timestamp of the last time this alarm changed status - time_t last_repeat; // the last time the alarm got repeated + time_t last_repeat; // the last time the alarm got repeated uint32_t times_repeat; // number of times the alarm got repeated time_t db_after; // the first timestamp evaluated by the db lookup @@ -150,49 +149,70 @@ struct rrdcalc { struct rrdcalc *rrdset_prev; struct rrdcalc *next; + struct rrdcalc *prev; }; -struct alert_config { - char *alarm; - char *template_key; - char *os; - char *host; - char *on; - char *families; - char *plugin; - char *module; - char *charts; - char *lookup; - char *calc; - char *warn; - char *crit; - char *every; - char *green; - char *red; - char *exec; - char *to; - char *units; - char *info; - char *classification; - char *component; - char *type; - char *delay; - char *options; - char *repeat; - char *host_labels; +#define rrdcalc_name(rc) string2str((rc)->name) +#define rrdcalc_chart_name(rc) string2str((rc)->chart) +#define rrdcalc_exec(rc) string2str((rc)->exec) +#define rrdcalc_recipient(rc) string2str((rc)->recipient) +#define rrdcalc_classification(rc) string2str((rc)->classification) +#define rrdcalc_component(rc) string2str((rc)->component) +#define rrdcalc_type(rc) string2str((rc)->type) +#define rrdcalc_plugin_match(rc) string2str((rc)->plugin_match) +#define rrdcalc_module_match(rc) string2str((rc)->module_match) +#define rrdcalc_source(rc) string2str((rc)->source) +#define rrdcalc_units(rc) string2str((rc)->units) +#define rrdcalc_original_info(rc) string2str((rc)->original_info) +#define rrdcalc_info(rc) string2str((rc)->info) +#define rrdcalc_dimensions(rc) string2str((rc)->dimensions) +#define rrdcalc_foreachdim(rc) string2str((rc)->foreachdim) +#define rrdcalc_host_labels(rc) string2str((rc)->host_labels) + +#define foreach_rrdcalc_in_rrdset(st, rc) \ + DOUBLE_LINKED_LIST_FOREACH_FORWARD((st)->alarms, rc, rrdset_prev, rrdset_next) + +#define foreach_rrdcalc_in_rrdhost(host, rc) \ + DOUBLE_LINKED_LIST_FOREACH_FORWARD((host)->host_alarms, rc, prev, next) + +struct alert_config { + STRING *alarm; + STRING *template_key; + STRING *os; + STRING *host; + STRING *on; + STRING *families; + STRING *plugin; + STRING *module; + STRING *charts; + STRING *lookup; + STRING *calc; + STRING *warn; + STRING *crit; + STRING *every; + STRING *green; + STRING *red; + STRING *exec; + STRING *to; + STRING *units; + STRING *info; + STRING *classification; + STRING *component; + STRING *type; + STRING *delay; + STRING *options; + STRING *repeat; + STRING *host_labels; + + STRING *p_db_lookup_dimensions; + STRING *p_db_lookup_method; - char *p_db_lookup_dimensions; - char *p_db_lookup_method; uint32_t p_db_lookup_options; int32_t p_db_lookup_after; int32_t p_db_lookup_before; int32_t p_update_every; }; -extern int alarm_isrepeating(RRDHOST *host, uint32_t alarm_id); -extern int alarm_entry_isrepeating(RRDHOST *host, ALARM_ENTRY *ae); -extern RRDCALC *alarm_max_last_repeat(RRDHOST *host, char *alarm_name, uint32_t hash); - #define RRDCALC_HAS_DB_LOOKUP(rc) ((rc)->after) extern void rrdsetcalc_link_matching(RRDSET *st); @@ -204,13 +224,13 @@ extern const char *rrdcalc_status2string(RRDCALC_STATUS status); extern void rrdcalc_free(RRDCALC *rc); extern void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc); -extern int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name); -extern uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id); +extern int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name); +extern uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id); extern RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart); extern RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const char *name, const char *dimension); extern void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc); extern void dimension_remove_pipe_comma(char *str); -extern char *alarm_name_with_dim(char *name, size_t namelen, const char *dim, size_t dimlen); +extern char *alarm_name_with_dim(const char *name, size_t namelen, const char *dim, size_t dimlen); extern void rrdcalc_update_rrdlabels(RRDSET *st); extern void rrdcalc_labels_unlink(); diff --git a/database/rrdcalctemplate.c b/database/rrdcalctemplate.c index 3f9804b936..4ba96082ac 100644 --- a/database/rrdcalctemplate.c +++ b/database/rrdcalctemplate.c @@ -12,30 +12,30 @@ * @param st is the chart where the alarm will be attached. */ void rrdcalctemplate_check_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) { - if(rt->hash_context != st->hash_context || strcmp(rt->context, st->context) != 0) + if(rt->context != st->context) return; - if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, st->name)) + if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, rrdset_name(st))) return; - if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, st->family)) + if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, rrdset_family(st))) return; - if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, st->module_name)) + if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, rrdset_module_name(st))) return; - if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, st->plugin_name)) + if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, rrdset_plugin_name(st))) return; if(host->host_labels && rt->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(host->host_labels, rt->host_labels_pattern, '=')) return; - RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id); + RRDCALC *rc = rrdcalc_create_from_template(host, rt, rrdset_id(st)); if (unlikely(!rc)) - info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, host->hostname); + info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rrdcalctemplate_name(rt), rrdset_id(st), rrdhost_hostname(host)); #ifdef NETDATA_INTERNAL_CHECKS - else if (rc->rrdset != st && !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late - error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart ? rc->chart : "NOCHART", rc->name, st->id); + else if (rc->rrdset != st && !rc->foreachdim) //When we have a template with foreachdim, the child will be added to the index late + error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st)); #endif } @@ -43,10 +43,7 @@ void rrdcalctemplate_link_matching(RRDSET *st) { RRDHOST *host = st->rrdhost; RRDCALCTEMPLATE *rt; - for(rt = host->templates; rt ; rt = rt->next) - rrdcalctemplate_check_conditions_and_link(rt, st, host); - - for(rt = host->alarms_template_with_foreach; rt ; rt = rt->next) + foreach_rrdcalctemplate_in_rrdhost(host, rt) rrdcalctemplate_check_conditions_and_link(rt, st, host); } @@ -57,31 +54,31 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) { expression_free(rt->warning); expression_free(rt->critical); - freez(rt->family_match); + string_freez(rt->family_match); simple_pattern_free(rt->family_pattern); - freez(rt->plugin_match); + string_freez(rt->plugin_match); simple_pattern_free(rt->plugin_pattern); - freez(rt->module_match); + string_freez(rt->module_match); simple_pattern_free(rt->module_pattern); - freez(rt->charts_match); + string_freez(rt->charts_match); simple_pattern_free(rt->charts_pattern); - freez(rt->name); - freez(rt->exec); - freez(rt->recipient); - freez(rt->classification); - freez(rt->component); - freez(rt->type); - freez(rt->context); - freez(rt->source); - freez(rt->units); - freez(rt->info); - freez(rt->dimensions); - freez(rt->foreachdim); - freez(rt->host_labels); + string_freez(rt->name); + string_freez(rt->exec); + string_freez(rt->recipient); + string_freez(rt->classification); + string_freez(rt->component); + string_freez(rt->type); + string_freez(rt->context); + string_freez(rt->source); + string_freez(rt->units); + string_freez(rt->info); + string_freez(rt->dimensions); + string_freez(rt->foreachdim); + string_freez(rt->host_labels); simple_pattern_free(rt->spdim); simple_pattern_free(rt->host_labels_pattern); freez(rt); @@ -90,21 +87,9 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) { inline void rrdcalctemplate_unlink_and_free(RRDHOST *host, RRDCALCTEMPLATE *rt) { if(unlikely(!rt)) return; - debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname); + debug(D_HEALTH, "Health removing template '%s' of host '%s'", rrdcalctemplate_name(rt), rrdhost_hostname(host)); - if(host->templates == rt) { - host->templates = rt->next; - } - else { - RRDCALCTEMPLATE *t; - for (t = host->templates; t && t->next != rt; t = t->next ) ; - if(t) { - t->next = rt->next; - rt->next = NULL; - } - else - error("Cannot find RRDCALCTEMPLATE '%s' linked in host '%s'", rt->name, host->hostname); - } + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->alarms_templates, rt, prev, next); rrdcalctemplate_free(rt); } diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h index 51aa330542..cbb23df4d3 100644 --- a/database/rrdcalctemplate.h +++ b/database/rrdcalctemplate.h @@ -9,35 +9,34 @@ // these are to be applied to charts found dynamically // based on their context. struct rrdcalctemplate { - char *name; - uint32_t hash_name; uuid_t config_hash_id; - char *exec; - char *recipient; + STRING *name; - char *classification; - char *component; - char *type; + STRING *exec; + STRING *recipient; - char *context; - uint32_t hash_context; + STRING *classification; + STRING *component; + STRING *type; - char *family_match; + STRING *context; + + STRING *family_match; SIMPLE_PATTERN *family_pattern; - char *plugin_match; + STRING *plugin_match; SIMPLE_PATTERN *plugin_pattern; - char *module_match; + STRING *module_match; SIMPLE_PATTERN *module_pattern; - char *charts_match; + STRING *charts_match; SIMPLE_PATTERN *charts_pattern; - char *source; // the source of this alarm - char *units; // the units of the alarm - char *info; // a short description of the alarm + STRING *source; // the source of this alarm + STRING *units; // the units of the alarm + STRING *info; // a short description of the alarm int update_every; // update frequency for the alarm @@ -48,8 +47,8 @@ struct rrdcalctemplate { // ------------------------------------------------------------------------ // database lookup settings - char *dimensions; // the chart dimensions - char *foreachdim; // the group of dimensions that the lookup will be applied. + STRING *dimensions; // the chart dimensions + STRING *foreachdim; // the group of dimensions that the lookup will be applied. SIMPLE_PATTERN *spdim; // used if and only if there is a simple pattern for the chart. int foreachcounter; // the number of alarms created with foreachdim, this also works as an id of the // children @@ -74,7 +73,7 @@ struct rrdcalctemplate { // ------------------------------------------------------------------------ // Labels settings - char *host_labels; // the label read from an alarm file + STRING *host_labels; // the label read from an alarm file SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels // ------------------------------------------------------------------------ @@ -85,8 +84,29 @@ struct rrdcalctemplate { EVAL_EXPRESSION *critical; struct rrdcalctemplate *next; + struct rrdcalctemplate *prev; }; +#define foreach_rrdcalctemplate_in_rrdhost(host, rt) \ + DOUBLE_LINKED_LIST_FOREACH_FORWARD((host)->alarms_templates, rt, prev, next) + +#define rrdcalctemplate_name(rt) string2str((rt)->name) +#define rrdcalctemplate_exec(rt) string2str((rt)->exec) +#define rrdcalctemplate_recipient(rt) string2str((rt)->recipient) +#define rrdcalctemplate_classification(rt) string2str((rt)->classification) +#define rrdcalctemplate_component(rt) string2str((rt)->component) +#define rrdcalctemplate_type(rt) string2str((rt)->type) +#define rrdcalctemplate_family_match(rt) string2str((rt)->family_match) +#define rrdcalctemplate_plugin_match(rt) string2str((rt)->plugin_match) +#define rrdcalctemplate_module_match(rt) string2str((rt)->module_match) +#define rrdcalctemplate_charts_match(rt) string2str((rt)->charts_match) +#define rrdcalctemplate_units(rt) string2str((rt)->units) +#define rrdcalctemplate_info(rt) string2str((rt)->info) +#define rrdcalctemplate_source(rt) string2str((rt)->source) +#define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions) +#define rrdcalctemplate_foreachdim(rt) string2str((rt)->foreachdim) +#define rrdcalctemplate_host_labels(rt) string2str((rt)->host_labels) + #define RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) ((rt)->after) extern void rrdcalctemplate_link_matching(RRDSET *st); diff --git a/database/rrdcontext.c b/database/rrdcontext.c index 71dd513b59..b3af8b44cc 100644 --- a/database/rrdcontext.c +++ b/database/rrdcontext.c @@ -14,7 +14,6 @@ int rrdcontext_enabled = CONFIG_BOOLEAN_YES; #define RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY 10 // #define LOG_TRANSITIONS 1 -// #define LOG_RRDINSTANCES 1 typedef enum { RRD_FLAG_NONE = 0, @@ -49,6 +48,9 @@ typedef enum { RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 28), // this context changed because of a db rotation RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 29), // this context is not used anymore RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS = (1 << 30), // this context is not used anymore + + // DO NOT ADD (1 << 31) or bigger! + // runtime error: left shift of 1 by 31 places cannot be represented in type 'int' } RRD_FLAGS; #define RRD_FLAG_ALL_UPDATE_REASONS ( \ @@ -85,10 +87,6 @@ typedef enum { #define RRD_FLAGS_PREVENTING_DELETIONS ( \ RRD_FLAG_QUEUED \ |RRD_FLAG_COLLECTED \ - |RRD_FLAG_UPDATE_REASON_LOAD_SQL \ - |RRD_FLAG_UPDATE_REASON_NEW_OBJECT \ - |RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT \ - |RRD_FLAG_UPDATE_REASON_CHANGED_LINKING \ ) #define rrd_flag_set_updated(obj, reason) (obj)->flags |= (RRD_FLAG_UPDATED | (reason)) @@ -295,6 +293,8 @@ static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reaso #define rrdcontext_version_hash(host) rrdcontext_version_hash_with_callback(host, NULL, false, NULL) static uint64_t rrdcontext_version_hash_with_callback(RRDHOST *host, void (*callback)(RRDCONTEXT *, bool, void *), bool snapshot, void *bundle); +static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker); +static void rrdcontext_garbage_collect(void); void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc); #define rrdcontext_lock(rc) netdata_mutex_lock(&((rc)->mutex)) @@ -303,9 +303,9 @@ void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc); // ---------------------------------------------------------------------------- // Updates triggers -static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate); -static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate); -static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force); +static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate, const char *function); +static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate, const char *function); +static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force, const char *function); // ---------------------------------------------------------------------------- // visualizing flags @@ -354,109 +354,46 @@ static void rrd_reasons_to_buffer(RRD_FLAGS flags, BUFFER *wb) { // logging of all data collected #ifdef LOG_TRANSITIONS -static void log_transition(STRING *metric, STRING *instance, STRING *context, RRD_FLAGS flags, const char *msg) { +static void log_transition(RRDMETRIC *rm, RRDINSTANCE *ri, RRDCONTEXT *rc, const char *function) { BUFFER *wb = buffer_create(1000); + const char *triggered_on = "triggered on "; - buffer_sprintf(wb, "RRD TRANSITION: context '%s'", string2str(context)); + buffer_sprintf(wb, "RRD TRANSITION: %s() ", function); - if(instance) - buffer_sprintf(wb, ", instance '%s'", string2str(instance)); + if(rm) { + buffer_sprintf(wb, "%smetric '%s' of ", triggered_on, string2str(rm->id)); + triggered_on = ""; + } - if(metric) - buffer_sprintf(wb, ", metric '%s'", string2str(metric)); + if(ri) { + buffer_sprintf(wb, "%sinstance '%s' of ", triggered_on, string2str(ri->id)); + triggered_on = ""; + } - buffer_sprintf(wb, ", triggered by %s: ", msg); + buffer_sprintf(wb, "%scontext '%s' ", triggered_on, string2str(rc->id)); + RRD_FLAGS flags = rc->flags; + const char *we_are = "context"; + if(ri) { + flags = ri->flags; + we_are = "instance"; + } + if(rm) { + flags = rm->flags; + we_are = "metric"; + } + + buffer_sprintf(wb, "%s flags: ", we_are); rrd_flags_to_buffer(flags, wb); - buffer_strcat(wb, ", reasons: "); - + buffer_strcat(wb, ", having reasons: "); rrd_reasons_to_buffer(flags, wb); internal_error(true, "%s", buffer_tostring(wb)); buffer_free(wb); } #else -#define log_transition(metric, instance, context, flags, msg) debug_dummy() -#endif - -#ifdef LOG_RRDINSTANCES -static void rrdinstance_log(RRDINSTANCE *ri, const char *msg) { - char uuid[UUID_STR_LEN]; - - uuid_unparse(ri->uuid, uuid); - - BUFFER *wb = buffer_create(1000); - - buffer_sprintf(wb, - "RRDINSTANCE: %s id '%s' (host '%s'), uuid '%s', name '%s', context '%s', title '%s', units '%s', family '%s', priority %zu, chart type '%s', update every %d, rrdset '%s', flags %s%s%s%s%s%s%s%s, first_time_t %ld, last_time_t %ld", - msg, - string2str(ri->id), - ri->rc->rrdhost->hostname, - uuid, - string2str(ri->name), - string2str(ri->rc->id), - string2str(ri->title), - string2str(ri->units), - string2str(ri->family), - ri->priority, - rrdset_type_name(ri->chart_type), - ri->update_every, - ri->rrdset?ri->rrdset->id:"NONE", - ri->flags & RRD_FLAG_DELETED ?"DELETED ":"", - ri->flags & RRD_FLAG_UPDATED ?"UPDATED ":"", - rrd_flag_is_collected(ri) ?"COLLECTED ":"", - rrd_flag_is_archived(ri) ?"ARCHIVED ":"", - ri->flags & RRD_FLAG_OWNLABELS ?"OWNLABELS ":"", - ri->flags & RRD_FLAG_LIVE_RETENTION ?"LIVE ":"", - ri->flags & RRD_FLAG_QUEUED ?"QUEUED ":"", - ri->flags & RRD_FLAG_DONT_TRIGGER ?"BLOCKED ":"", - ri->first_time_t, - ri->last_time_t - ); - - buffer_strcat(wb, ", update reasons: { "); - for(int i = 0, added = 0; rrdcontext_reasons[i].name ;i++) - if(ri->flags & rrdcontext_reasons[i].flag) { - if(added) buffer_strcat(wb, ", "); - buffer_strcat(wb, rrdcontext_reasons[i].name); - added++; - } - buffer_strcat(wb, " }"); - - buffer_strcat(wb, ", labels: { "); - if(ri->rrdlabels) { - if(!rrdlabels_to_buffer(ri->rrdlabels, wb, "", "=", "'", ", ", NULL, NULL, NULL, NULL)) - buffer_strcat(wb, "EMPTY }"); - else - buffer_strcat(wb, " }"); - } - else - buffer_strcat(wb, "NONE }"); - - buffer_strcat(wb, ", metrics: { "); - if(ri->rrdmetrics) { - RRDMETRIC *v; - int i = 0; - dfe_start_read((DICTIONARY *)ri->rrdmetrics, v) { - buffer_sprintf(wb, "%s%s", i?",":"", v_name); - i++; - } - dfe_done(v); - - if(!i) - buffer_strcat(wb, "EMPTY }"); - else - buffer_strcat(wb, " }"); - } - else - buffer_strcat(wb, "NONE }"); - - internal_error(true, "%s", buffer_tostring(wb)); - buffer_free(wb); -} -#else -#define rrdinstance_log(ir, msg) debug_dummy() +#define log_transition(rm, ri, rc, function) debug_dummy() #endif // ---------------------------------------------------------------------------- @@ -577,7 +514,7 @@ static void rrdmetric_conflict_callback(const char *id __maybe_unused, void *old char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; uuid_unparse(rm->uuid, uuid1); uuid_unparse(rm_new->uuid, uuid2); - internal_error(true, "RRDMETRIC: '%s' is linked to RRDDIM '%s' but they have different UUIDs. RRDMETRIC has '%s', RRDDIM has '%s'", string2str(rm->id), rm->rrddim->id, uuid1, uuid2); + internal_error(true, "RRDMETRIC: '%s' is linked to RRDDIM '%s' but they have different UUIDs. RRDMETRIC has '%s', RRDDIM has '%s'", string2str(rm->id), rrddim_id(rm->rrddim), uuid1, uuid2); } if(rm->rrddim != rm_new->rrddim) @@ -616,7 +553,7 @@ static void rrdmetric_conflict_callback(const char *id __maybe_unused, void *old static void rrdmetric_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) { RRDMETRIC *rm = value; - rrdmetric_trigger_updates(rm, false, true); + rrdmetric_trigger_updates(rm, false, true, __FUNCTION__); } static void rrdmetrics_create(RRDINSTANCE *ri) { @@ -649,8 +586,8 @@ static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) { if(likely(rm->rrddim)) return false; - if((now_realtime_usec() - rm->created_ut) < 600 * USEC_PER_SEC) - return false; + //if((now_realtime_usec() - rm->created_ut) < 600 * USEC_PER_SEC) + // return false; rrdmetric_update_retention(rm); if(rm->first_time_t || rm->last_time_t) @@ -659,38 +596,37 @@ static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) { return true; } -static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate) { +static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate, const char *function __maybe_unused) { if(likely(!force && !(rm->flags & RRD_FLAG_UPDATED))) return; - if(unlikely(rrd_flag_is_collected(rm) && !rm->rrddim)) - rrd_flag_set_archived(rm); + // logs and statistics + log_transition(rm, rm->ri, rm->ri->rc, function); + rrdcontext_triggered_update_on_rrdmetric(); - if(unlikely((rm->flags & RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD) && rrd_flag_is_collected(rm))) - rrd_flag_set_archived(rm); + if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rm->flags & RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD)) + rrd_flag_set_archived(rm); rrdmetric_update_retention(rm); - if(unlikely(escalate && rm->flags & RRD_FLAG_UPDATED && !(rm->ri->flags & RRD_FLAG_DONT_PROCESS))) { - log_transition(rm->id, rm->ri->id, rm->ri->rc->id, rm->flags, "RRDMETRIC"); - rrdinstance_trigger_updates(rm->ri, true, true); - } + if(unlikely(escalate && rm->flags & RRD_FLAG_UPDATED)) + rrdinstance_trigger_updates(rm->ri, true, true, __FUNCTION__); } static inline void rrdmetric_from_rrddim(RRDDIM *rd) { if(unlikely(!rd->rrdset)) - fatal("RRDMETRIC: rrddim '%s' does not have a rrdset.", rd->id); + fatal("RRDMETRIC: rrddim '%s' does not have a rrdset.", rrddim_id(rd)); if(unlikely(!rd->rrdset->rrdhost)) - fatal("RRDMETRIC: rrdset '%s' does not have a rrdhost", rd->rrdset->id); + fatal("RRDMETRIC: rrdset '%s' does not have a rrdhost", rrdset_id(rd->rrdset)); if(unlikely(!rd->rrdset->rrdinstance)) - fatal("RRDMETRIC: rrdset '%s' does not have a rrdinstance", rd->rrdset->id); + fatal("RRDMETRIC: rrdset '%s' does not have a rrdinstance", rrdset_id(rd->rrdset)); RRDINSTANCE *ri = rrdinstance_acquired_value(rd->rrdset->rrdinstance); RRDMETRIC trm = { - .id = string_strdupz(rd->id), - .name = string_strdupz(rd->name), + .id = string_dup(rd->id), + .name = string_dup(rd->name), .flags = RRD_FLAG_NONE, .rrddim = rd, }; @@ -707,14 +643,14 @@ static inline void rrdmetric_from_rrddim(RRDDIM *rd) { #define rrddim_get_rrdmetric(rd) rrddim_get_rrdmetric_with_trace(rd, __FUNCTION__) static inline RRDMETRIC *rrddim_get_rrdmetric_with_trace(RRDDIM *rd, const char *function) { if(unlikely(!rd->rrdmetric)) { - error("RRDMETRIC: RRDDIM '%s' is not linked to an RRDMETRIC at %s()", rd->id, function); + error("RRDMETRIC: RRDDIM '%s' is not linked to an RRDMETRIC at %s()", rrddim_id(rd), function); return NULL; } RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdmetric); if(unlikely(rm->rrddim != rd)) - fatal("RRDMETRIC: '%s' is not linked to RRDDIM '%s' at %s()", string2str(rm->id), rd->id, function); + fatal("RRDMETRIC: '%s' is not linked to RRDDIM '%s' at %s()", string2str(rm->id), rrddim_id(rd), function); return rm; } @@ -727,7 +663,7 @@ static inline void rrdmetric_rrddim_is_freed(RRDDIM *rd) { rrd_flag_set_archived(rm); rm->rrddim = NULL; - rrdmetric_trigger_updates(rm, false, true); + rrdmetric_trigger_updates(rm, false, true, __FUNCTION__); rrdmetric_release(rd->rrdmetric); rd->rrdmetric = NULL; } @@ -736,12 +672,12 @@ static inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) { RRDMETRIC *rm = rrddim_get_rrdmetric(rd); if(unlikely(!rm)) return; - if(unlikely(rd->flags & (RRDDIM_FLAG_ARCHIVED | RRDDIM_FLAG_OBSOLETE))) { + if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED|RRDDIM_FLAG_OBSOLETE))) { if(unlikely(rrd_flag_is_collected(rm))) rrd_flag_set_archived(rm); } - rrdmetric_trigger_updates(rm, false, true); + rrdmetric_trigger_updates(rm, false, true, __FUNCTION__); } static inline void rrdmetric_collected_rrddim(RRDDIM *rd) { @@ -751,7 +687,7 @@ static inline void rrdmetric_collected_rrddim(RRDDIM *rd) { if(unlikely(!rrd_flag_is_collected(rm))) rrd_flag_set_collected(rm); - rrdmetric_trigger_updates(rm, false, true); + rrdmetric_trigger_updates(rm, false, true, __FUNCTION__); } // ---------------------------------------------------------------------------- @@ -807,7 +743,7 @@ static void rrdinstance_insert_callback(const char *id __maybe_unused, void *val } if(ri->rrdset) { - if(unlikely((ri->rrdset->flags & RRDSET_FLAG_HIDDEN) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart))) + if(unlikely((rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart))) ri->flags |= RRD_FLAG_HIDDEN; else ri->flags &= ~RRD_FLAG_HIDDEN; @@ -818,7 +754,6 @@ static void rrdinstance_insert_callback(const char *id __maybe_unused, void *val ri->flags |= RRD_FLAG_HIDDEN; rrdmetrics_create(ri); - rrdinstance_log(ri, "INSERT"); // signal the react callback to do the job rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_NEW_OBJECT); @@ -829,8 +764,6 @@ static void rrdinstance_delete_callback(const char *id, void *value, void *data) RRDCONTEXT *rc = data; (void)rc; RRDINSTANCE *ri = (RRDINSTANCE *)value; - rrdinstance_log(ri, "DELETE"); - internal_error(ri->rrdset, "RRDINSTANCE: '%s' is freed but there is a RRDSET linked to it.", string2str(ri->id)); rrdinstance_free(ri); @@ -858,7 +791,7 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; uuid_unparse(ri->uuid, uuid1); uuid_unparse(*ri->rrdset->chart_uuid, uuid2); - internal_error(true, "RRDINSTANCE: '%s' is linked to RRDSET '%s' but they have different UUIDs. RRDINSTANCE has '%s', RRDSET has '%s'", string2str(ri->id), ri->rrdset->id, uuid1, uuid2); + internal_error(true, "RRDINSTANCE: '%s' is linked to RRDSET '%s' but they have different UUIDs. RRDINSTANCE has '%s', RRDSET has '%s'", string2str(ri->id), rrdset_id(ri->rrdset), uuid1, uuid2); } if(ri->name != ri_new->name) { @@ -920,7 +853,7 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o } if(ri->rrdset) { - if(unlikely((ri->rrdset->flags & RRDSET_FLAG_HIDDEN) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart))) + if(unlikely((rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart))) ri->flags |= RRD_FLAG_HIDDEN; else ri->flags &= ~RRD_FLAG_HIDDEN; @@ -934,8 +867,6 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o if(ri->flags & RRD_FLAG_UPDATED) ri->flags |= RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT; - rrdinstance_log(ri, "CONFLICT"); - // free the new one rrdinstance_free(ri_new); @@ -945,7 +876,7 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o static void rrdinstance_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) { RRDINSTANCE *ri = value; - rrdinstance_trigger_updates(ri, false, true); + rrdinstance_trigger_updates(ri, false, true, __FUNCTION__); } void rrdinstances_create(RRDCONTEXT *rc) { @@ -993,10 +924,14 @@ static inline bool rrdinstance_should_be_deleted(RRDINSTANCE *ri) { return true; } -static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate) { +static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate, const char *function __maybe_unused) { if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS)) return; if(unlikely(!force && !(ri->flags & RRD_FLAG_UPDATED))) return; + // logs and stats + log_transition(NULL, ri, ri->rc, function); + rrdcontext_triggered_update_on_rrdinstance(); + if(likely(ri->rrdset)) { if(unlikely(ri->rrdset->priority != ri->priority)) { ri->priority = ri->rrdset->priority; @@ -1015,10 +950,10 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala time_t min_first_time_t = LONG_MAX, max_last_time_t = 0; size_t metrics_active = 0, metrics_deleted = 0; bool live_retention = true, currently_collected = false; - { + if(dictionary_stats_entries(ri->rrdmetrics) > 0) { RRDMETRIC *rm; dfe_start_read((DICTIONARY *)ri->rrdmetrics, rm) { - if(!(rm->flags & RRD_FLAG_LIVE_RETENTION)) + if(unlikely(!(rm->flags & RRD_FLAG_LIVE_RETENTION))) live_retention = false; if (unlikely((rrdmetric_should_be_deleted(rm)))) { @@ -1027,7 +962,7 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala continue; } - if(rm->flags & RRD_FLAG_COLLECTED) + if(rm->flags & RRD_FLAG_COLLECTED && rm->first_time_t) currently_collected = true; metrics_active++; @@ -1043,9 +978,9 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala dfe_done(rm); } - if(live_retention && !(ri->flags & RRD_FLAG_LIVE_RETENTION)) + if(unlikely(live_retention && !(ri->flags & RRD_FLAG_LIVE_RETENTION))) ri->flags |= RRD_FLAG_LIVE_RETENTION; - else if(!live_retention && (ri->flags & RRD_FLAG_LIVE_RETENTION)) + else if(unlikely(!live_retention && (ri->flags & RRD_FLAG_LIVE_RETENTION))) ri->flags &= ~RRD_FLAG_LIVE_RETENTION; if(unlikely(!metrics_active)) { @@ -1080,7 +1015,7 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T); } - if(unlikely(live_retention)) + if(likely(live_retention)) rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION); } else { @@ -1103,18 +1038,16 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala } } - if(unlikely(escalate && ri->flags & RRD_FLAG_UPDATED && !(ri->rc->flags & RRD_FLAG_DONT_PROCESS))) { - log_transition(NULL, ri->id, ri->rc->id, ri->flags, "RRDINSTANCE"); - rrdcontext_trigger_updates(ri->rc, true); - } + if(unlikely(escalate && ri->flags & RRD_FLAG_UPDATED)) + rrdcontext_trigger_updates(ri->rc, true, __FUNCTION__); } static inline void rrdinstance_from_rrdset(RRDSET *st) { RRDCONTEXT trc = { - .id = string_strdupz(st->context), - .title = string_strdupz(st->title), - .units = string_strdupz(st->units), - .family = string_strdupz(st->family), + .id = string_dup(st->context), + .title = string_dup(st->title), + .units = string_dup(st->units), + .family = string_dup(st->family), .priority = st->priority, .chart_type = st->chart_type, .flags = RRD_FLAG_NONE, @@ -1125,11 +1058,11 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) { RRDCONTEXT *rc = rrdcontext_acquired_value(rca); RRDINSTANCE tri = { - .id = string_strdupz(st->id), - .name = string_strdupz(st->name), - .units = string_strdupz(st->units), - .family = string_strdupz(st->family), - .title = string_strdupz(st->title), + .id = string_dup(st->id), + .name = string_dup(st->name), + .units = string_dup(st->units), + .family = string_dup(st->family), + .title = string_dup(st->title), .chart_type = st->chart_type, .priority = st->priority, .update_every = st->update_every, @@ -1192,7 +1125,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) { ri_old->flags &= ~RRD_FLAG_DONT_PROCESS; rc_old->flags &= ~RRD_FLAG_DONT_PROCESS; - rrdinstance_trigger_updates(ri_old, true, true); + rrdinstance_trigger_updates(ri_old, true, true, __FUNCTION__); ri_old->flags |= RRD_FLAG_DONT_PROCESS; rrdinstance_release(ria_old); @@ -1205,10 +1138,10 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) { rc_old->first_time_t = 0; rc_old->last_time_t = 0; rrdcontext_unlock(rc_old); - rrdcontext_trigger_updates(rc_old, true); + rrdcontext_trigger_updates(rc_old, true, __FUNCTION__); } else - rrdcontext_trigger_updates(rc_old, true); + rrdcontext_trigger_updates(rc_old, true, __FUNCTION__); */ rrdcontext_release(rca_old); @@ -1223,14 +1156,14 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) { #define rrdset_get_rrdinstance(st) rrdset_get_rrdinstance_with_trace(st, __FUNCTION__); static inline RRDINSTANCE *rrdset_get_rrdinstance_with_trace(RRDSET *st, const char *function) { if(unlikely(!st->rrdinstance)) { - error("RRDINSTANCE: RRDSET '%s' is not linked to an RRDINSTANCE at %s()", st->id, function); + error("RRDINSTANCE: RRDSET '%s' is not linked to an RRDINSTANCE at %s()", rrdset_id(st), function); return NULL; } RRDINSTANCE *ri = rrdinstance_acquired_value(st->rrdinstance); if(unlikely(ri->rrdset != st)) - fatal("RRDINSTANCE: '%s' is not linked to RRDSET '%s' at %s()", string2str(ri->id), st->id, function); + fatal("RRDINSTANCE: '%s' is not linked to RRDSET '%s' at %s()", string2str(ri->id), rrdset_id(st), function); return ri; } @@ -1250,7 +1183,7 @@ static inline void rrdinstance_rrdset_is_freed(RRDSET *st) { ri->rrdset = NULL; ri->flags &= ~RRD_FLAG_DONT_PROCESS; - rrdinstance_trigger_updates(ri, false, true); + rrdinstance_trigger_updates(ri, false, true, __FUNCTION__); ri->flags |= RRD_FLAG_DONT_PROCESS; rrdinstance_release(st->rrdinstance); @@ -1267,26 +1200,26 @@ static inline void rrdinstance_updated_rrdset_name(RRDSET *st) { RRDINSTANCE *ri = rrdset_get_rrdinstance(st); if(unlikely(!ri)) return; - STRING *old = ri->name; - ri->name = string_strdupz(st->name); + if(st->name != ri->name) { + STRING *old = ri->name; + ri->name = string_dup(st->name); + string_freez(old); - if(ri->name != old) rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_NAME); - - string_freez(old); - - rrdinstance_trigger_updates(ri, false, true); + rrdinstance_trigger_updates(ri, false, true, __FUNCTION__); + } } static inline void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, RRDSET *st) { - if(unlikely(st->flags & (RRDSET_FLAG_ARCHIVED | RRDSET_FLAG_OBSOLETE))) - rrd_flag_set_archived(ri); + if(unlikely(ri->rrdset != st)) + fatal("RRDCONTEXT: instance '%s' is not linked to chart '%s' on host '%s'", + string2str(ri->id), rrdset_id(st), rrdhost_hostname(st->rrdhost)); - if(unlikely((st->flags & RRDSET_FLAG_HIDDEN) && !(ri->flags & RRD_FLAG_HIDDEN))) { + if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)) && !(ri->flags & RRD_FLAG_HIDDEN))) { ri->flags |= RRD_FLAG_HIDDEN; rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS); } - else if(unlikely(!(st->flags & RRDSET_FLAG_HIDDEN) && (ri->flags & RRD_FLAG_HIDDEN))) { + else if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)) && (ri->flags & RRD_FLAG_HIDDEN))) { ri->flags &= ~RRD_FLAG_HIDDEN; rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS); } @@ -1296,10 +1229,13 @@ static inline void rrdinstance_updated_rrdset_flags(RRDSET *st) { RRDINSTANCE *ri = rrdset_get_rrdinstance(st); if(unlikely(!ri)) return; + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED|RRDSET_FLAG_OBSOLETE))) + rrd_flag_set_archived(ri); + rrdinstance_updated_rrdset_flags_no_action(ri, st); ri->flags &= ~RRD_FLAG_DONT_PROCESS; - rrdinstance_trigger_updates(ri, false, true); + rrdinstance_trigger_updates(ri, false, true, __FUNCTION__); ri->flags |= RRD_FLAG_DONT_PROCESS; } @@ -1309,13 +1245,16 @@ static inline void rrdinstance_collected_rrdset(RRDSET *st) { rrdinstance_updated_rrdset_flags_no_action(ri, st); - if(unlikely(!rrd_flag_is_collected(ri))) - rrd_flag_set_collected(ri); + if(dictionary_stats_entries(ri->rrdmetrics) > 0) { - if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS)) - ri->flags &= ~RRD_FLAG_DONT_PROCESS; + if(unlikely(!rrd_flag_is_collected(ri))) + rrd_flag_set_collected(ri); - rrdinstance_trigger_updates(ri, false, true); + if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS)) + ri->flags &= ~RRD_FLAG_DONT_PROCESS; + + rrdinstance_trigger_updates(ri, false, true, __FUNCTION__); + } } // ---------------------------------------------------------------------------- @@ -1595,7 +1534,7 @@ static void rrdcontext_conflict_callback(const char *id, void *oldv, void *newv, static void rrdcontext_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) { RRDCONTEXT *rc = (RRDCONTEXT *)value; - rrdcontext_trigger_updates(rc, false); + rrdcontext_trigger_updates(rc, false, __FUNCTION__); } void rrdhost_create_rrdcontexts(RRDHOST *host) { @@ -1649,17 +1588,21 @@ static inline bool rrdcontext_should_be_deleted(RRDCONTEXT *rc) { return true; } -static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force) { +static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force, const char *function __maybe_unused) { if(unlikely(rc->flags & RRD_FLAG_DONT_PROCESS)) return; if(unlikely(!force && !(rc->flags & RRD_FLAG_UPDATED))) return; + // logs and stats + log_transition(NULL, NULL, rc, function); + rrdcontext_triggered_update_on_rrdcontext(); + rrdcontext_lock(rc); size_t min_priority = LONG_MAX; time_t min_first_time_t = LONG_MAX, max_last_time_t = 0; size_t instances_active = 0, instances_deleted = 0; bool live_retention = true, currently_collected = false, hidden = true; - { + if(dictionary_stats_entries(rc->rrdinstances) > 0) { RRDINSTANCE *ri; dfe_start_read(rc->rrdinstances, ri) { if(likely(!(ri->flags & RRD_FLAG_HIDDEN))) @@ -1674,7 +1617,7 @@ static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force) { continue; } - if(ri->flags & RRD_FLAG_COLLECTED) + if(ri->flags & RRD_FLAG_COLLECTED && ri->first_time_t) currently_collected = true; internal_error(rc->units != ri->units, @@ -1768,8 +1711,6 @@ static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force) { } if(unlikely(rc->flags & RRD_FLAG_UPDATED)) { - log_transition(NULL, NULL, rc->id, rc->flags, "RRDCONTEXT"); - if(check_if_cloud_version_changed_unsafe(rc, false)) { rc->version = rrdcontext_get_next_version(rc); @@ -1945,7 +1886,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) { if(rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) { info("RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', while node '%s' has an active context streaming.", - cmd->claim_id, cmd->node_id, host->hostname); + cmd->claim_id, cmd->node_id, rrdhost_hostname(host)); // disable it temporarily, so that our worker will not attempt to send messages in parallel rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS); @@ -1955,7 +1896,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) { if(cmd->version_hash != our_version_hash) { error("RRDCONTEXT: received version hash %"PRIu64" for host '%s', does not match our version hash %"PRIu64". Sending snapshot of all contexts.", - cmd->version_hash, host->hostname, our_version_hash); + cmd->version_hash, rrdhost_hostname(host), our_version_hash); #ifdef ENABLE_ACLK // prepare the snapshot @@ -1977,11 +1918,11 @@ void rrdcontext_hub_checkpoint_command(void *ptr) { #endif } - internal_error(true, "RRDCONTEXT: host '%s' enabling streaming of contexts", host->hostname); + internal_error(true, "RRDCONTEXT: host '%s' enabling streaming of contexts", rrdhost_hostname(host)); rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS); char node_str[UUID_STR_LEN]; uuid_unparse_lower(*host->node_id, node_str); - log_access("ACLK REQ [%s (%s)]: STREAM CONTEXTS ENABLED", node_str, host->hostname); + log_access("ACLK REQ [%s (%s)]: STREAM CONTEXTS ENABLED", node_str, rrdhost_hostname(host)); } void rrdcontext_hub_stop_streaming_command(void *ptr) { @@ -2006,12 +1947,12 @@ void rrdcontext_hub_stop_streaming_command(void *ptr) { if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) { error("RRDCONTEXT: received stop streaming command for claim id '%s', node id '%s', but node '%s' does not have active context streaming. Ignoring command.", - cmd->claim_id, cmd->node_id, host->hostname); + cmd->claim_id, cmd->node_id, rrdhost_hostname(host)); return; } - internal_error(true, "RRDCONTEXT: host '%s' disabling streaming of contexts", host->hostname); + internal_error(true, "RRDCONTEXT: host '%s' disabling streaming of contexts", rrdhost_hostname(host)); rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS); } @@ -2378,7 +2319,7 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions) { if(!host->rrdctx) { - error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, host->hostname); + error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, rrdhost_hostname(host)); return HTTP_RESP_NOT_FOUND; } @@ -2418,7 +2359,7 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions) { if(!host->rrdctx) { - error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, host->hostname); + error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, rrdhost_hostname(host)); return HTTP_RESP_NOT_FOUND; } @@ -2440,7 +2381,7 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, ",\n\t\"machine_guid\": \"%s\"" ",\n\t\"node_id\": \"%s\"" ",\n\t\"claim_id\": \"%s\"" - , host->hostname + , rrdhost_hostname(host) , host->machine_guid , node_uuid , host->aclk_state.claimed_id ? host->aclk_state.claimed_id : "" @@ -2529,7 +2470,7 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) { ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri); ctx_get_label_list(&ri->uuid, rrdinstance_load_clabel, ri); ri->flags &= ~RRD_FLAG_DONT_PROCESS; - rrdinstance_trigger_updates(ri, true, true); + rrdinstance_trigger_updates(ri, true, true, __FUNCTION__); // let the instance be in "don't process" mode // so that we process it once, when it is collected @@ -2568,9 +2509,11 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) { RRDCONTEXT *rc; dfe_start_read((DICTIONARY *)host->rrdctx, rc) { rc->flags &= ~RRD_FLAG_DONT_PROCESS; - rrdcontext_trigger_updates(rc, true); + rrdcontext_trigger_updates(rc, true, __FUNCTION__); } dfe_done(rc); + + rrdcontext_garbage_collect_single_host(host, false); } // ---------------------------------------------------------------------------- @@ -2679,19 +2622,18 @@ static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS r rrd_flag_set_updated(rm, reason); - rm->flags &= ~RRD_FLAG_DONT_PROCESS; - rrdmetric_trigger_updates(rm, true, false); + rrdmetric_trigger_updates(rm, true, false, __FUNCTION__); } dfe_done(rm); ri->flags &= ~RRD_FLAG_DONT_PROCESS; - rrdinstance_trigger_updates(ri, true, false); + rrdinstance_trigger_updates(ri, true, false, __FUNCTION__); ri->flags |= RRD_FLAG_DONT_PROCESS; } dfe_done(ri); rc->flags &= ~RRD_FLAG_DONT_PROCESS; - rrdcontext_trigger_updates(rc, true); + rrdcontext_trigger_updates(rc, true, __FUNCTION__); } static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, int job_id) { @@ -2727,50 +2669,88 @@ void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) { error("RRDCONTEXT: failed to delete context '%s' version %"PRIu64" from SQL.", rc->hub.id, rc->hub.version); } +static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker) { + + internal_error(true, "RRDCONTEXT: garbage collecting context structures of host '%s'", rrdhost_hostname(host)); + + RRDCONTEXT *rc; + dfe_start_write((DICTIONARY *)host->rrdctx, rc) { + if(worker) worker_is_busy(WORKER_JOB_CLEANUP); + + rrdcontext_lock(rc); + + RRDINSTANCE *ri; + dfe_start_write(rc->rrdinstances, ri) { + RRDMETRIC *rm; + dfe_start_write(ri->rrdmetrics, rm) { + if(rrdmetric_should_be_deleted(rm)) { + if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE); + if(dictionary_del_having_write_lock(ri->rrdmetrics, string2str(rm->id)) != 0) + error("RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.", + string2str(rm->id), + string2str(ri->id), + string2str(rc->id), + rrdhost_hostname(host)); + else + internal_error( + true, + "RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', deleted from rrdmetrics dictionary.", + string2str(rm->id), + string2str(ri->id), + string2str(rc->id), + rrdhost_hostname(host)); + } + } + dfe_done(rm); + + if(rrdinstance_should_be_deleted(ri)) { + if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE); + if(dictionary_del_having_write_lock(rc->rrdinstances, string2str(ri->id)) != 0) + error("RRDCONTEXT: instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.", + string2str(ri->id), + string2str(rc->id), + rrdhost_hostname(host)); + else + internal_error( + true, + "RRDCONTEXT: instance '%s' of context '%s' of host '%s', deleted from rrdmetrics dictionary.", + string2str(ri->id), + string2str(rc->id), + rrdhost_hostname(host)); + } + } + dfe_done(ri); + + if(unlikely(rrdcontext_should_be_deleted(rc))) { + if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE); + rrdcontext_delete_from_sql_unsafe(rc); + + if(dictionary_del_having_write_lock((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0) + error("RRDCONTEXT: context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.", + string2str(rc->id), + rrdhost_hostname(host)); + else + internal_error( + true, + "RRDCONTEXT: context '%s' of host '%s', deleted from rrdmetrics dictionary.", + string2str(rc->id), + rrdhost_hostname(host)); + + fprintf(stderr, "RRDCONTEXT: deleted context '%s'", string2str(rc->id)); + } + + // the item is referenced in the dictionary + // so, it is still here to unlock, even if we have deleted it + rrdcontext_unlock(rc); + } + dfe_done(rc); +} + static void rrdcontext_garbage_collect(void) { rrd_rdlock(); RRDHOST *host; rrdhost_foreach_read(host) { - RRDCONTEXT *rc; - dfe_start_write((DICTIONARY *)host->rrdctx, rc) { - worker_is_busy(WORKER_JOB_CLEANUP); - - rrdcontext_lock(rc); - - if(unlikely(rrdcontext_should_be_deleted(rc))) { - worker_is_busy(WORKER_JOB_CLEANUP_DELETE); - rrdcontext_delete_from_sql_unsafe(rc); - - if(dictionary_del_having_write_lock((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0) - error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.", - string2str(rc->id), host->hostname); - } - else { - RRDINSTANCE *ri; - dfe_start_write(rc->rrdinstances, ri) { - if(rrdinstance_should_be_deleted(ri)) { - worker_is_busy(WORKER_JOB_CLEANUP_DELETE); - dictionary_del_having_write_lock(rc->rrdinstances, string2str(ri->id)); - } - else { - RRDMETRIC *rm; - dfe_start_write(ri->rrdmetrics, rm) { - if(rrdmetric_should_be_deleted(rm)) { - worker_is_busy(WORKER_JOB_CLEANUP_DELETE); - dictionary_del_having_write_lock(ri->rrdmetrics, string2str(rm->id)); - } - } - dfe_done(rm); - } - } - dfe_done(ri); - } - - // the item is referenced in the dictionary - // so, it is still here to unlock, even if we have deleted it - rrdcontext_unlock(rc); - } - dfe_done(rc); + rrdcontext_garbage_collect_single_host(host, true); } rrd_unlock(); } @@ -2895,7 +2875,7 @@ void *rrdcontext_main(void *ptr) { // delete it from the master dictionary if(dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0) error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.", - string2str(id), host->hostname); + string2str(id), rrdhost_hostname(host)); string_freez(id); } diff --git a/database/rrddim.c b/database/rrddim.c index 90165a2535..7a7a5802ef 100644 --- a/database/rrddim.c +++ b/database/rrddim.c @@ -10,48 +10,59 @@ // ---------------------------------------------------------------------------- // RRDDIM index -int rrddim_compare(void* a, void* b) { - if(((RRDDIM *)a)->hash < ((RRDDIM *)b)->hash) return -1; - else if(((RRDDIM *)a)->hash > ((RRDDIM *)b)->hash) return 1; - else return strcmp(((RRDDIM *)a)->id, ((RRDDIM *)b)->id); +static inline void rrddim_index_add(RRDSET *st, RRDDIM *rd) { + if(likely(dictionary_set(st->rrddim_root_index, string2str(rd->id), rd, sizeof(RRDDIM)) == rd)) { + rrddim_flag_set(rd, RRDDIM_FLAG_INDEXED_ID); + } + else { + rrddim_flag_clear(rd, RRDDIM_FLAG_INDEXED_ID); + error("RRDDIM: %s() attempted to index duplicate dimension with key '%s' of chart '%s' of host '%s'", __FUNCTION__, rrddim_id(rd), rrdset_id(st), rrdhost_hostname(st->rrdhost)); + } } -#define rrddim_index_add(st, rd) (RRDDIM *)avl_insert_lock(&((st)->dimensions_index), (avl_t *)(rd)) -#define rrddim_index_del(st,rd ) (RRDDIM *)avl_remove_lock(&((st)->dimensions_index), (avl_t *)(rd)) - -static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id, uint32_t hash) { - RRDDIM tmp = { - .id = id, - .hash = (hash)?hash:simple_hash(id) - }; - return (RRDDIM *)avl_search_lock(&(st->dimensions_index), (avl_t *) &tmp); +static inline void rrddim_index_del(RRDSET *st, RRDDIM *rd) { + if(rrddim_flag_check(rd, RRDDIM_FLAG_INDEXED_ID)) { + if (likely(dictionary_del(st->rrddim_root_index, string2str(rd->id)) == 0)) + rrddim_flag_clear(rd, RRDDIM_FLAG_INDEXED_ID); + else + error("RRDDIM: %s() attempted to delete non-indexed dimension with key '%s' of chart '%s' of host '%s'", __FUNCTION__, rrddim_id(rd), rrdset_id(st), rrdhost_hostname(st->rrdhost)); + } } +static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id) { + return dictionary_get(st->rrddim_root_index, id); +} // ---------------------------------------------------------------------------- // RRDDIM - find a dimension inline RRDDIM *rrddim_find(RRDSET *st, const char *id) { - debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", st->name, id); + debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", rrdset_name(st), id); - return rrddim_index_find(st, id, 0); + return rrddim_index_find(st, id); } +// This will not return dimensions that are archived +RRDDIM *rrddim_find_active(RRDSET *st, const char *id) { + RRDDIM *rd = rrddim_find(st, id); + + if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))) + return NULL; + + return rd; +} // ---------------------------------------------------------------------------- // RRDDIM rename a dimension inline int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) { - if(unlikely(!name || !*name || (rd->name && !strcmp(rd->name, name)))) + if(unlikely(!name || !*name || !strcmp(rrddim_name(rd), name))) return 0; - debug(D_RRD_CALLS, "rrddim_set_name() from %s.%s to %s.%s", st->name, rd->name, st->name, name); + debug(D_RRD_CALLS, "rrddim_set_name() from %s.%s to %s.%s", rrdset_name(st), rrddim_name(rd), rrdset_name(st), name); - if (rd->name) - freez((void *) rd->name); - - rd->name = strdupz(name); - rd->hash_name = simple_hash(rd->name); + string_freez(rd->name); + rd->name = rrd_string_strdupz(name); if (!st->state->is_ar_chart) rrddimvar_rename_all(rd); @@ -68,7 +79,7 @@ inline int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm) if(unlikely(rd->algorithm == algorithm)) return 0; - debug(D_RRD_CALLS, "Updating algorithm of dimension '%s/%s' from %s to %s", st->id, rd->name, rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm)); + debug(D_RRD_CALLS, "Updating algorithm of dimension '%s/%s' from %s to %s", rrdset_id(st), rrddim_name(rd), rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm)); rd->algorithm = algorithm; rd->exposed = 0; rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK); @@ -81,7 +92,7 @@ inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip if(unlikely(rd->multiplier == multiplier)) return 0; - debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, st->id, rd->name, rd->multiplier, multiplier); + debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, rrdset_id(st), rrddim_name(rd), rd->multiplier, multiplier); rd->multiplier = multiplier; rd->exposed = 0; rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK); @@ -94,7 +105,7 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor) if(unlikely(rd->divisor == divisor)) return 0; - debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, st->id, rd->name, rd->divisor, divisor); + debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, rrdset_id(st), rrddim_name(rd), rd->divisor, divisor); rd->divisor = divisor; rd->exposed = 0; rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK); @@ -107,31 +118,27 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor) // RRDDIM create a dimension void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) { - RRDCALC *rrdc; + RRDCALC *rc; - for (rrdc = host->alarms_with_foreach; rrdc ; rrdc = rrdc->next) { - if (simple_pattern_matches(rrdc->spdim, rd->id) || simple_pattern_matches(rrdc->spdim, rd->name)) { - if (rrdc->hash_chart == st->hash_name || !strcmp(rrdc->chart, st->name) || !strcmp(rrdc->chart, st->id)) { - char *name = alarm_name_with_dim(rrdc->name, strlen(rrdc->name), rd->name, strlen(rd->name)); - if(rrdcalc_exists(host, st->name, name, 0, 0)) { + for (rc = host->alarms_with_foreach; rc; rc = rc->next) { + if (simple_pattern_matches(rc->spdim, rrddim_id(rd)) || simple_pattern_matches(rc->spdim, rrddim_name(rd))) { + if (rc->chart == st->name || rc->chart == st->id) { + char *name = alarm_name_with_dim(rrdcalc_name(rc), string_strlen(rc->name), rrddim_name(rd), string_strlen(rd->name)); + if(rrdcalc_exists(host, rrdset_name(st), name)) { freez(name); continue; } netdata_rwlock_wrlock(&host->health_log.alarm_log_rwlock); - RRDCALC *child = rrdcalc_create_from_rrdcalc(rrdc, host, name, rd->name); + RRDCALC *child = rrdcalc_create_from_rrdcalc(rc, host, name, rrddim_name(rd)); netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); - if (child) { + if (child) rrdcalc_add_to_host(host, child); - RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl_t *)child); - if (rdcmp != child) { - error("Cannot insert the alarm index ID %s",child->name); - } - } + else { error("Cannot allocate a new alarm."); - rrdc->foreachcounter--; + rc->foreachcounter--; } } } @@ -157,15 +164,20 @@ time_t calc_dimension_liveness(RRDDIM *rd, time_t now) } #endif -RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, - collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) -{ +RRDDIM *rrddim_add_custom(RRDSET *st + , const char *id + , const char *name + , collected_number multiplier + , collected_number divisor + , RRD_ALGORITHM algorithm + , RRD_MEMORY_MODE memory_mode + ) { RRDHOST *host = st->rrdhost; rrdset_wrlock(st); RRDDIM *rd = rrddim_find(st, id); if(unlikely(rd)) { - debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:""); + debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", rrdset_id(st), name?name:""); int rc = rrddim_set_name(st, rd, name); rc += rrddim_set_algorithm(st, rd, algorithm); @@ -192,8 +204,8 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte } if (unlikely(rc)) { - debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id); - (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor, + debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rrddim_id(rd)); + (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); #ifdef ENABLE_ACLK queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec())); @@ -210,11 +222,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); rd = callocz(1, sizeof(RRDDIM)); - rd->id = strdupz(id); - rd->hash = simple_hash(rd->id); + rd->id = string_strdupz(id); - rd->name = (name && *name)?strdupz(name):strdupz(rd->id); - rd->hash_name = simple_hash(rd->name); + rd->name = (name && *name)?rrd_string_strdupz(name):string_dup(rd->id); rd->algorithm = algorithm; rd->multiplier = multiplier; @@ -231,7 +241,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte if(memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_SAVE) { if(!rrddim_memory_load_or_create_map_save(st, rd, memory_mode)) { - info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name, rd->name); + info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st), rrddim_name(rd)); memory_mode = RRD_MEMORY_MODE_RAM; } } @@ -242,7 +252,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1); if(!rd->db) { - info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", st->name, rd->name); + info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd)); memory_mode = RRD_MEMORY_MODE_ALLOC; } else rd->memsize = entries * sizeof(storage_number); @@ -261,6 +271,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte #ifdef ENABLE_ACLK rd->aclk_live_status = -1; #endif + (void) find_dimension_uuid(st, rd, &(rd->metric_uuid)); // initialize the db tiers @@ -284,10 +295,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte } if(!initialized) - error("Failed to initialize all db tiers for chart '%s', dimension '%s", st->name, rd->name); + error("Failed to initialize all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); if(!rd->tiers[0]) - error("Failed to initialize the first db tier for chart '%s', dimension '%s", st->name, rd->name); + error("Failed to initialize the first db tier for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); } store_active_dimension(&rd->metric_uuid); @@ -303,22 +314,19 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte } if(!initialized) - error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", st->name, rd->name); + error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); } - // append this dimension - if(!st->dimensions) - st->dimensions = rd; - else { + if(st->dimensions) { RRDDIM *td = st->dimensions; if(td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor)) { if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { #ifdef NETDATA_INTERNAL_CHECKS info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").", - rd->name, - st->name, - host->hostname, + rrddim_name(rd), + rrdset_name(st), + rrdhost_hostname(host), rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(td->algorithm), rd->multiplier, td->multiplier, rd->divisor, td->divisor @@ -327,19 +335,18 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS); } } - - for(; td->next; td = td->next) ; - td->next = rd; } + // append this dimension + DOUBLE_LINKED_LIST_APPEND_UNSAFE(st->dimensions, rd, prev, next); + if(host->health_enabled && !st->state->is_ar_chart) { rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT); rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT); rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); } - if(unlikely(rrddim_index_add(st, rd) != rd)) - error("RRDDIM: INTERNAL ERROR: attempt to index duplicate dimension '%s' on chart '%s'", rd->id, st->id); + rrddim_index_add(st, rd); rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM); rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS); @@ -360,7 +367,7 @@ void rrddim_free(RRDSET *st, RRDDIM *rd) rrdcontext_removed_rrddim(rd); ml_delete_dimension(rd); - debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name); + debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd)); if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { @@ -382,24 +389,12 @@ void rrddim_free(RRDSET *st, RRDDIM *rd) } } - if(rd == st->dimensions) - st->dimensions = rd->next; - else { - RRDDIM *i; - for (i = st->dimensions; i && i->next != rd; i = i->next) ; - - if (i && i->next == rd) - i->next = rd->next; - else - error("Request to free dimension '%s.%s' but it is not linked.", st->id, rd->name); - } - rd->next = NULL; + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->dimensions, rd, prev, next); while(rd->variables) rrddimvar_free(rd->variables); - if(unlikely(rrddim_index_del(st, rd) != rd)) - error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id); + rrddim_index_del(st, rd); // free(rd->annotations); //#ifdef ENABLE_ACLK @@ -428,8 +423,8 @@ void rrddim_free(RRDSET *st, RRDDIM *rd) freez(rd->db); } - freez((void *)rd->id); - freez((void *)rd->name); + string_freez(rd->id); + string_freez(rd->name); freez(rd); } @@ -438,13 +433,13 @@ void rrddim_free(RRDSET *st, RRDDIM *rd) // RRDDIM - set dimension options int rrddim_hide(RRDSET *st, const char *id) { - debug(D_RRD_CALLS, "rrddim_hide() for chart %s, dimension %s", st->name, id); + debug(D_RRD_CALLS, "rrddim_hide() for chart %s, dimension %s", rrdset_name(st), id); RRDHOST *host = st->rrdhost; RRDDIM *rd = rrddim_find(st, id); if(unlikely(!rd)) { - error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, host->hostname); + error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host)); return 1; } if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) @@ -457,12 +452,12 @@ int rrddim_hide(RRDSET *st, const char *id) { } int rrddim_unhide(RRDSET *st, const char *id) { - debug(D_RRD_CALLS, "rrddim_unhide() for chart %s, dimension %s", st->name, id); + debug(D_RRD_CALLS, "rrddim_unhide() for chart %s, dimension %s", rrdset_name(st), id); RRDHOST *host = st->rrdhost; RRDDIM *rd = rrddim_find(st, id); if(unlikely(!rd)) { - error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, host->hostname); + error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host)); return 1; } if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) @@ -475,10 +470,10 @@ int rrddim_unhide(RRDSET *st, const char *id) { } inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) { - debug(D_RRD_CALLS, "rrddim_is_obsolete() for chart %s, dimension %s", st->name, rd->name); + debug(D_RRD_CALLS, "rrddim_is_obsolete() for chart %s, dimension %s", rrdset_name(st), rrddim_name(rd)); if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))) { - info("Cannot obsolete already archived dimension %s from chart %s", rd->name, st->name); + info("Cannot obsolete already archived dimension %s from chart %s", rrddim_name(rd), rrdset_name(st)); return; } rrddim_flag_set(rd, RRDDIM_FLAG_OBSOLETE); @@ -487,7 +482,7 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) { } inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) { - debug(D_RRD_CALLS, "rrddim_isnot_obsolete() for chart %s, dimension %s", st->name, rd->name); + debug(D_RRD_CALLS, "rrddim_isnot_obsolete() for chart %s, dimension %s", rrdset_name(st), rrddim_name(rd)); rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE); rrdcontext_updated_rrddim_flags(rd); @@ -497,9 +492,7 @@ inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) { // RRDDIM - collect values for a dimension inline collected_number rrddim_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM *rd, collected_number value) { - debug(D_RRD_CALLS, "rrddim_set_by_pointer() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value); - - rrdcontext_collected_rrddim(rd); + debug(D_RRD_CALLS, "rrddim_set_by_pointer() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, rrdset_name(st), rrddim_name(rd), value); now_realtime_timeval(&rd->last_collected_time); rd->collected_value = value; @@ -519,7 +512,7 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) RRDHOST *host = st->rrdhost; RRDDIM *rd = rrddim_find(st, id); if(unlikely(!rd)) { - error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, host->hostname); + error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host)); return 0; } @@ -627,7 +620,7 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO char filename[FILENAME_MAX + 1]; char fullfilename[FILENAME_MAX + 1]; - rrdset_strncpyz_name(filename, rd->id, FILENAME_MAX); + rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX); snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename); rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(fullfilename, size, diff --git a/database/rrddimvar.c b/database/rrddimvar.c index 3c2ed75e57..1726612645 100644 --- a/database/rrddimvar.c +++ b/database/rrddimvar.c @@ -16,64 +16,64 @@ static inline void rrddimvar_free_variables(RRDDIMVAR *rs) { // CHART VARIABLES FOR THIS DIMENSION - rrdvar_free(host, &st->rrdvar_root_index, rs->var_local_id); + rrdvar_free(host, st->rrdvar_root_index, rs->var_local_id); rs->var_local_id = NULL; - rrdvar_free(host, &st->rrdvar_root_index, rs->var_local_name); + rrdvar_free(host, st->rrdvar_root_index, rs->var_local_name); rs->var_local_name = NULL; // FAMILY VARIABLES FOR THIS DIMENSION - rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_id); + rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_id); rs->var_family_id = NULL; - rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_name); + rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_name); rs->var_family_name = NULL; - rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_contextid); + rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_contextid); rs->var_family_contextid = NULL; - rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_contextname); + rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_contextname); rs->var_family_contextname = NULL; // HOST VARIABLES FOR THIS DIMENSION - rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_chartidid); + rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartidid); rs->var_host_chartidid = NULL; - rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_chartidname); + rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartidname); rs->var_host_chartidname = NULL; - rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_chartnameid); + rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartnameid); rs->var_host_chartnameid = NULL; - rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_chartnamename); + rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartnamename); rs->var_host_chartnamename = NULL; // KEYS - freez(rs->key_id); + string_freez(rs->key_id); rs->key_id = NULL; - freez(rs->key_name); + string_freez(rs->key_name); rs->key_name = NULL; - freez(rs->key_fullidid); + string_freez(rs->key_fullidid); rs->key_fullidid = NULL; - freez(rs->key_fullidname); + string_freez(rs->key_fullidname); rs->key_fullidname = NULL; - freez(rs->key_contextid); + string_freez(rs->key_contextid); rs->key_contextid = NULL; - freez(rs->key_contextname); + string_freez(rs->key_contextname); rs->key_contextname = NULL; - freez(rs->key_fullnameid); + string_freez(rs->key_fullnameid); rs->key_fullnameid = NULL; - freez(rs->key_fullnamename); + string_freez(rs->key_fullnamename); rs->key_fullnamename = NULL; } @@ -88,29 +88,29 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // KEYS - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->id, rs->suffix); - rs->key_id = strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", string2str(rs->prefix), rrddim_id(rd), string2str(rs->suffix)); + rs->key_id = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix); - rs->key_name = strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", string2str(rs->prefix), rrddim_name(rd), string2str(rs->suffix)); + rs->key_name = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->key_id); - rs->key_fullidid = strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_id(st), string2str(rs->key_id)); + rs->key_fullidid = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->key_name); - rs->key_fullidname = strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_id(st), string2str(rs->key_name)); + rs->key_fullidname = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->context, rs->key_id); - rs->key_contextid = strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_context(st), string2str(rs->key_id)); + rs->key_contextid = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->context, rs->key_name); - rs->key_contextname = strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_context(st), string2str(rs->key_name)); + rs->key_contextname = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->key_id); - rs->key_fullnameid = strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_name(st), string2str(rs->key_id)); + rs->key_fullnameid = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->key_name); - rs->key_fullnamename = strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_name(st), string2str(rs->key_name)); + rs->key_fullnamename = string_strdupz(buffer); // CHART VARIABLES FOR THIS DIMENSION // ----------------------------------- @@ -119,8 +119,8 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $id // - $name - rs->var_local_id = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_local_name = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_local_id = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_local_name = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); // FAMILY VARIABLES FOR THIS DIMENSION // ----------------------------------- @@ -131,10 +131,10 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $chart-context.id // - $chart-context.name - rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_family_id = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_family_name = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_family_contextid = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_family_contextname = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); // HOST VARIABLES FOR THIS DIMENSION // ----------------------------------- @@ -145,33 +145,32 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $chart-name.id // - $chart-name.name - rs->var_host_chartidid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_host_chartidname = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_host_chartnameid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnameid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_host_chartnamename = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnamename, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_host_chartidid = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullidid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_host_chartidname = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullidname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_host_chartnameid = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullnameid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + rs->var_host_chartnamename = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullnamename, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); } RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options) { RRDSET *st = rd->rrdset; (void)st; - debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:""); + debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", rrdset_id(st), rrdset_name(st), rrddim_id(rd), (prefix)?prefix:"", rrddim_name(rd), (suffix)?suffix:""); if(!prefix) prefix = ""; if(!suffix) suffix = ""; RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR)); - rs->prefix = strdupz(prefix); - rs->suffix = strdupz(suffix); + rs->prefix = string_strdupz(prefix); + rs->suffix = string_strdupz(suffix); rs->type = type; rs->value = value; rs->options = options; rs->rrddim = rd; - rs->next = rd->variables; - rd->variables = rs; + DOUBLE_LINKED_LIST_PREPEND_UNSAFE(rd->variables, rs, prev, next); rrddimvar_create_variables(rs); @@ -182,7 +181,7 @@ void rrddimvar_rename_all(RRDDIM *rd) { RRDSET *st = rd->rrdset; (void)st; - debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name); + debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", rrdset_id(st), rrdset_name(st), rrddim_id(rd), rrddim_name(rd)); RRDDIMVAR *rs, *next = rd->variables; while((rs = next)) { @@ -193,25 +192,13 @@ void rrddimvar_rename_all(RRDDIM *rd) { void rrddimvar_free(RRDDIMVAR *rs) { RRDDIM *rd = rs->rrddim; - RRDSET *st = rd->rrdset; - debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", st->id, st->name, rd->id, rd->name, rs->prefix, rs->suffix); + debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset), rrddim_id(rd), rrddim_name(rd), string2str(rs->prefix), string2str(rs->suffix)); rrddimvar_free_variables(rs); - if(rd->variables == rs) { - debug(D_VARIABLES, "RRDDIMSET removing first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name); - rd->variables = rs->next; - } - else { - debug(D_VARIABLES, "RRDDIMSET removing non-first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name); - RRDDIMVAR *t; - for (t = rd->variables; t && t->next != rs; t = t->next) ; - if(!t) error("RRDDIMVAR '%s' not found in dimension '%s/%s' variables linked list", rs->key_name, st->id, rd->id); - else t->next = rs->next; - } + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rd->variables, rs, prev, next); - freez(rs->prefix); - freez(rs->suffix); + string_freez(rs->prefix); + string_freez(rs->suffix); freez(rs); } - diff --git a/database/rrddimvar.h b/database/rrddimvar.h index 3494824be1..6af2a14974 100644 --- a/database/rrddimvar.h +++ b/database/rrddimvar.h @@ -11,17 +11,17 @@ // This means, there will be no speed penalty for using // these variables struct rrddimvar { - char *prefix; - char *suffix; + STRING *prefix; + STRING *suffix; - char *key_id; // dimension id - char *key_name; // dimension name - char *key_contextid; // context + dimension id - char *key_contextname; // context + dimension name - char *key_fullidid; // chart type.chart id + dimension id - char *key_fullidname; // chart type.chart id + dimension name - char *key_fullnameid; // chart type.chart name + dimension id - char *key_fullnamename; // chart type.chart name + dimension name + STRING *key_id; // dimension id + STRING *key_name; // dimension name + STRING *key_contextid; // context + dimension id + STRING *key_contextname; // context + dimension name + STRING *key_fullidid; // chart type.chart id + dimension id + STRING *key_fullidname; // chart type.chart id + dimension name + STRING *key_fullnameid; // chart type.chart name + dimension id + STRING *key_fullnamename; // chart type.chart name + dimension name RRDVAR_TYPE type; void *value; @@ -44,6 +44,7 @@ struct rrddimvar { struct rrddim *rrddim; struct rrddimvar *next; + struct rrddimvar *prev; }; diff --git a/database/rrdfamily.c b/database/rrdfamily.c index 3d91c37887..438f1a07cb 100644 --- a/database/rrdfamily.c +++ b/database/rrdfamily.c @@ -6,37 +6,38 @@ // ---------------------------------------------------------------------------- // RRDFAMILY index -int rrdfamily_compare(void *a, void *b) { - if(((RRDFAMILY *)a)->hash_family < ((RRDFAMILY *)b)->hash_family) return -1; - else if(((RRDFAMILY *)a)->hash_family > ((RRDFAMILY *)b)->hash_family) return 1; - else return strcmp(((RRDFAMILY *)a)->family, ((RRDFAMILY *)b)->family); +static inline RRDFAMILY *rrdfamily_index_add(RRDHOST *host, RRDFAMILY *rc) { + return dictionary_set(host->rrdfamily_root_index, string2str(rc->family), rc, sizeof(RRDFAMILY)); } -#define rrdfamily_index_add(host, rc) (RRDFAMILY *)avl_insert_lock(&((host)->rrdfamily_root_index), (avl_t *)(rc)) -#define rrdfamily_index_del(host, rc) (RRDFAMILY *)avl_remove_lock(&((host)->rrdfamily_root_index), (avl_t *)(rc)) - -static RRDFAMILY *rrdfamily_index_find(RRDHOST *host, const char *id, uint32_t hash) { - RRDFAMILY tmp; - tmp.family = id; - tmp.hash_family = (hash)?hash:simple_hash(tmp.family); - - return (RRDFAMILY *)avl_search_lock(&(host->rrdfamily_root_index), (avl_t *) &tmp); +static inline RRDFAMILY *rrdfamily_index_del(RRDHOST *host, RRDFAMILY *rc) { + dictionary_del(host->rrdfamily_root_index, string2str(rc->family)); + return rc; } +static inline RRDFAMILY *rrdfamily_index_find(RRDHOST *host, const char *id) { + return dictionary_get(host->rrdfamily_root_index, id); +} + +// ---------------------------------------------------------------------------- +// RRDFAMILY management + RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id) { - RRDFAMILY *rc = rrdfamily_index_find(host, id, 0); + RRDFAMILY *rc = rrdfamily_index_find(host, id); if(!rc) { rc = callocz(1, sizeof(RRDFAMILY)); - rc->family = strdupz(id); - rc->hash_family = simple_hash(rc->family); + rc->family = string_strdupz(id); - // initialize the variables index - avl_init_lock(&rc->rrdvar_root_index, rrdvar_compare); + rc->rrdvar_root_index = dictionary_create( + DICTIONARY_FLAG_NAME_LINK_DONT_CLONE + |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE + |DICTIONARY_FLAG_DONT_OVERWRITE_VALUE + ); RRDFAMILY *ret = rrdfamily_index_add(host, rc); if(ret != rc) - error("RRDFAMILY: INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", rc->family, (ret)?ret->family:"NONE"); + error("RRDFAMILY: INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", string2str(rc->family), (ret)?string2str(ret->family):"NONE"); } rc->use_count++; @@ -48,12 +49,13 @@ void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc) { if(!rc->use_count) { RRDFAMILY *ret = rrdfamily_index_del(host, rc); if(ret != rc) - error("RRDFAMILY: INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", rc->family, (ret)?ret->family:"NONE"); + error("RRDFAMILY: INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", string2str(rc->family), (ret)?string2str(ret->family):"NONE"); else { - debug(D_RRD_CALLS, "RRDFAMILY: Cleaning up remaining family variables for host '%s', family '%s'", host->hostname, rc->family); - rrdvar_free_remaining_variables(host, &rc->rrdvar_root_index); + debug(D_RRD_CALLS, "RRDFAMILY: Cleaning up remaining family variables for host '%s', family '%s'", rrdhost_hostname(host), string2str(rc->family)); + rrdvar_free_remaining_variables(host, rc->rrdvar_root_index); - freez((void *) rc->family); + dictionary_destroy(rc->rrdvar_root_index); + string_freez(rc->family); freez(rc); } } diff --git a/database/rrdhost.c b/database/rrdhost.c index 7f4bd95ba6..fe765ca90f 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -43,107 +43,150 @@ bool is_storage_engine_shared(STORAGE_INSTANCE *engine) { // ---------------------------------------------------------------------------- -// RRDHOST index +// RRDHOST indexes management -int rrdhost_compare(void* a, void* b) { - if(((RRDHOST *)a)->hash_machine_guid < ((RRDHOST *)b)->hash_machine_guid) return -1; - else if(((RRDHOST *)a)->hash_machine_guid > ((RRDHOST *)b)->hash_machine_guid) return 1; - else return strcmp(((RRDHOST *)a)->machine_guid, ((RRDHOST *)b)->machine_guid); +static DICTIONARY *rrdhost_root_index = NULL; +static DICTIONARY *rrdhost_root_index_hostname = NULL; + +static inline void rrdhost_init() { + if(unlikely(!rrdhost_root_index)) { + rrdhost_root_index = dictionary_create( + DICTIONARY_FLAG_NAME_LINK_DONT_CLONE + | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE + | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE + ); + } + + if(unlikely(!rrdhost_root_index_hostname)) { + rrdhost_root_index_hostname = dictionary_create( + DICTIONARY_FLAG_NAME_LINK_DONT_CLONE + | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE + | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE + ); + } } -avl_tree_lock rrdhost_root_index = { - .avl_tree = { NULL, rrdhost_compare }, - .rwlock = AVL_LOCK_INITIALIZER -}; +// ---------------------------------------------------------------------------- +// RRDHOST index by UUID -RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash) { - debug(D_RRDHOST, "Searching in index for host with guid '%s'", guid); - - RRDHOST tmp; - strncpyz(tmp.machine_guid, guid, GUID_LEN); - tmp.hash_machine_guid = (hash)?hash:simple_hash(tmp.machine_guid); - - return (RRDHOST *)avl_search_lock(&(rrdhost_root_index), (avl_t *) &tmp); +inline long rrdhost_hosts_available(void) { + return dictionary_stats_entries(rrdhost_root_index); } -RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash) { +inline RRDHOST *rrdhost_find_by_guid(const char *guid) { + return dictionary_get(rrdhost_root_index, guid); +} + +static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) { + RRDHOST *ret_machine_guid = dictionary_set(rrdhost_root_index, host->machine_guid, host, sizeof(RRDHOST)); + if(ret_machine_guid == host) + rrdhost_flag_set(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID); + else { + rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID); + error("RRDHOST: %s() host with machine guid '%s' is already indexed", __FUNCTION__, host->machine_guid); + } + + return host; +} + +static void rrdhost_index_del_by_guid(RRDHOST *host) { + if(rrdhost_flag_check(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID)) { + if(dictionary_del(rrdhost_root_index, host->machine_guid) != 0) + error("RRDHOST: %s() failed to delete machine guid '%s' from index", __FUNCTION__, host->machine_guid); + + rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID); + } +} + +// ---------------------------------------------------------------------------- +// RRDHOST index by hostname + +inline RRDHOST *rrdhost_find_by_hostname(const char *hostname) { if(unlikely(!strcmp(hostname, "localhost"))) return localhost; - if(unlikely(!hash)) hash = simple_hash(hostname); - - rrd_rdlock(); - RRDHOST *host; - rrdhost_foreach_read(host) { - if(unlikely((hash == host->hash_hostname && !strcmp(hostname, host->hostname)))) { - rrd_unlock(); - return host; - } - } - rrd_unlock(); - - return NULL; + return dictionary_get(rrdhost_root_index_hostname, hostname); } -#define rrdhost_index_add(rrdhost) (RRDHOST *)avl_insert_lock(&(rrdhost_root_index), (avl_t *)(rrdhost)) -#define rrdhost_index_del(rrdhost) (RRDHOST *)avl_remove_lock(&(rrdhost_root_index), (avl_t *)(rrdhost)) +static inline RRDHOST *rrdhost_index_add_hostname(RRDHOST *host) { + if(!host->hostname) return host; + RRDHOST *ret_hostname = dictionary_set(rrdhost_root_index_hostname, rrdhost_hostname(host), host, sizeof(RRDHOST)); + if(ret_hostname == host) + rrdhost_flag_set(host, RRDHOST_FLAG_INDEXED_HOSTNAME); + else { + rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_HOSTNAME); + error("RRDHOST: %s() host with hostname '%s' is already indexed", __FUNCTION__, rrdhost_hostname(host)); + } + + return host; +} + +static inline void rrdhost_index_del_hostname(RRDHOST *host) { + if(unlikely(!host->hostname)) return; + + if(rrdhost_flag_check(host, RRDHOST_FLAG_INDEXED_HOSTNAME)) { + if(dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host)) != 0) + error("RRDHOST: %s() failed to delete hostname '%s' from index", __FUNCTION__, rrdhost_hostname(host)); + + rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_HOSTNAME); + } +} // ---------------------------------------------------------------------------- // RRDHOST - internal helpers static inline void rrdhost_init_tags(RRDHOST *host, const char *tags) { - if(host->tags && tags && !strcmp(host->tags, tags)) + if(host->tags && tags && !strcmp(rrdhost_tags(host), tags)) return; - void *old = (void *)host->tags; - host->tags = (tags && *tags)?strdupz(tags):NULL; - freez(old); + STRING *old = host->tags; + host->tags = string_strdupz((tags && *tags)?tags:NULL); + string_freez(old); } static inline void rrdhost_init_hostname(RRDHOST *host, const char *hostname) { - if(host->hostname && hostname && !strcmp(host->hostname, hostname)) + if(unlikely(hostname && !*hostname)) hostname = NULL; + + if(host->hostname && hostname && !strcmp(rrdhost_hostname(host), hostname)) return; - void *old = host->hostname; - host->hostname = strdupz(hostname?hostname:"localhost"); - host->hash_hostname = simple_hash(host->hostname); - freez(old); + rrdhost_index_del_hostname(host); + + STRING *old = host->hostname; + host->hostname = string_strdupz(hostname?hostname:"localhost"); + string_freez(old); + + rrdhost_index_add_hostname(host); } static inline void rrdhost_init_os(RRDHOST *host, const char *os) { - if(host->os && os && !strcmp(host->os, os)) + if(host->os && os && !strcmp(rrdhost_os(host), os)) return; - void *old = (void *)host->os; - host->os = strdupz(os?os:"unknown"); - freez(old); + STRING *old = host->os; + host->os = string_strdupz(os?os:"unknown"); + string_freez(old); } static inline void rrdhost_init_timezone(RRDHOST *host, const char *timezone, const char *abbrev_timezone, int32_t utc_offset) { - if (host->timezone && timezone && !strcmp(host->timezone, timezone) && host->abbrev_timezone && abbrev_timezone && - !strcmp(host->abbrev_timezone, abbrev_timezone) && host->utc_offset == utc_offset) + if (host->timezone && timezone && !strcmp(rrdhost_timezone(host), timezone) && host->abbrev_timezone && abbrev_timezone && + !strcmp(rrdhost_abbrev_timezone(host), abbrev_timezone) && host->utc_offset == utc_offset) return; - void *old = (void *)host->timezone; - host->timezone = strdupz((timezone && *timezone)?timezone:"unknown"); - freez(old); + STRING *old = host->timezone; + host->timezone = string_strdupz((timezone && *timezone)?timezone:"unknown"); + string_freez(old); old = (void *)host->abbrev_timezone; - host->abbrev_timezone = strdupz((abbrev_timezone && *abbrev_timezone) ? abbrev_timezone : "UTC"); + host->abbrev_timezone = string_strdupz((abbrev_timezone && *abbrev_timezone) ? abbrev_timezone : "UTC"); freez(old); host->utc_offset = utc_offset; } -static inline void rrdhost_init_machine_guid(RRDHOST *host, const char *machine_guid) { - strncpy(host->machine_guid, machine_guid, GUID_LEN); - host->machine_guid[GUID_LEN] = '\0'; - host->hash_machine_guid = simple_hash(host->machine_guid); -} - -void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *hostname, - const char *registry_hostname, const char *guid, const char *os, const char *tags, +void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, + const char *registry_hostname, const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *program_name, const char *program_version) { @@ -151,18 +194,13 @@ void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory host->rrd_update_every = update_every; host->rrd_memory_mode = memory_mode; - rrdhost_init_hostname(host, hostname); - - rrdhost_init_machine_guid(host, guid); - rrdhost_init_os(host, os); rrdhost_init_timezone(host, tzone, abbrev_tzone, utc_offset); rrdhost_init_tags(host, tags); - host->program_name = strdupz((program_name && *program_name) ? program_name : "unknown"); - host->program_version = strdupz((program_version && *program_version) ? program_version : "unknown"); - - host->registry_hostname = strdupz((registry_hostname && *registry_hostname) ? registry_hostname : host->hostname); + host->program_name = string_strdupz((program_name && *program_name) ? program_name : "unknown"); + host->program_version = string_strdupz((program_version && *program_version) ? program_version : "unknown"); + host->registry_hostname = string_strdupz((registry_hostname && *registry_hostname) ? registry_hostname : rrdhost_hostname(host)); } // ---------------------------------------------------------------------------- @@ -202,9 +240,14 @@ RRDHOST *rrdhost_create(const char *hostname, int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy); RRDHOST *host = callocz(1, sizeof(RRDHOST)); - set_host_properties(host, (update_every > 0)?update_every:1, memory_mode, hostname, registry_hostname, guid, os, + strncpy(host->machine_guid, guid, GUID_LEN); + host->machine_guid[GUID_LEN] = '\0'; + + set_host_properties(host, (update_every > 0)?update_every:1, memory_mode, registry_hostname, os, tags, timezone, abbrev_timezone, utc_offset, program_name, program_version); + rrdhost_init_hostname(host, hostname); + host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries); host->health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled; @@ -237,10 +280,10 @@ RRDHOST *rrdhost_create(const char *hostname, host->system_info = system_info; - avl_init_lock(&(host->rrdset_root_index), rrdset_compare); - avl_init_lock(&(host->rrdset_root_index_name), rrdset_compare_name); - avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare); - avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare); + host->rrdset_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + host->rrdset_root_index_name = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + host->rrdfamily_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + host->rrdvar_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); if(config_get_boolean(CONFIG_SECTION_DB, "delete obsolete charts files", 1)) rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); @@ -250,8 +293,6 @@ RRDHOST *rrdhost_create(const char *hostname, host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never"); host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never"); - avl_init_lock(&(host->alarms_idx_health_log), alarm_compare_id); - avl_init_lock(&(host->alarms_idx_name), alarm_compare_name); // ------------------------------------------------------------------------ // initialize health variables @@ -264,7 +305,7 @@ RRDHOST *rrdhost_create(const char *hostname, long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max); if(n < 10) { - error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", host->hostname, n, host->health_log.max); + error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", rrdhost_hostname(host), n, host->health_log.max); config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max); } else @@ -293,7 +334,7 @@ RRDHOST *rrdhost_create(const char *hostname, host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) { int r = mkdir(host->cache_dir, 0775); if(r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", host->hostname, host->cache_dir); + error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->cache_dir); } snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_varlib_dir, host->machine_guid); @@ -302,7 +343,7 @@ RRDHOST *rrdhost_create(const char *hostname, if(host->health_enabled) { int r = mkdir(host->varlib_dir, 0775); if(r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", host->hostname, host->varlib_dir); + error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir); } } @@ -311,15 +352,15 @@ RRDHOST *rrdhost_create(const char *hostname, snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir); int r = mkdir(filename, 0775); if(r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", host->hostname, filename); + error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename); } snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir); host->health_log_filename = strdupz(filename); snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir); - host->health_default_exec = strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename)); - host->health_default_recipient = strdupz("root"); + host->health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename)); + host->health_default_recipient = string_strdupz("root"); // ------------------------------------------------------------------------ @@ -331,10 +372,9 @@ RRDHOST *rrdhost_create(const char *hostname, rrdhost_unlock(host); } - RRDHOST *t = rrdhost_index_add(host); - + RRDHOST *t = rrdhost_index_add_by_guid(host); if(t != host) { - error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", host->hostname, host->machine_guid, t->hostname, t->machine_guid); + error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", rrdhost_hostname(host), host->machine_guid, rrdhost_hostname(t), t->machine_guid); rrdhost_free(host, 1); return NULL; } @@ -376,7 +416,7 @@ RRDHOST *rrdhost_create(const char *hostname, snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", host->cache_dir); ret = mkdir(dbenginepath, 0775); if (ret != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath); + error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), dbenginepath); else ret = 0; // succeed if (is_legacy) { // initialize legacy dbengine instance as needed @@ -403,7 +443,7 @@ RRDHOST *rrdhost_create(const char *hostname, if (ret) { // check legacy or multihost initialization success error( "Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.", - host->hostname, host->machine_guid, host->cache_dir); + rrdhost_hostname(host), host->machine_guid, host->cache_dir); rrdhost_free(host, 1); host = NULL; //rrd_hosts_available++; //TODO: maybe we want this? @@ -426,17 +466,10 @@ RRDHOST *rrdhost_create(const char *hostname, // ------------------------------------------------------------------------ // link it and add it to the index - if(is_localhost) { - host->next = localhost; - localhost = host; - } - else { - if(localhost) { - host->next = localhost->next; - localhost->next = host; - } - else localhost = host; - } + if(is_localhost) + DOUBLE_LINKED_LIST_PREPEND_UNSAFE(localhost, host, prev, next); + else + DOUBLE_LINKED_LIST_APPEND_UNSAFE(localhost, host, prev, next); // ------------------------------------------------------------------------ // init new ML host and update system_info to let upstreams know @@ -466,14 +499,14 @@ RRDHOST *rrdhost_create(const char *hostname, ", health_log '%s'" ", alarms default handler '%s'" ", alarms default recipient '%s'" - , host->hostname - , host->registry_hostname + , rrdhost_hostname(host) + , rrdhost_registry_hostname(host) , host->machine_guid - , host->os - , host->timezone - , (host->tags)?host->tags:"" - , host->program_name - , host->program_version + , rrdhost_os(host) + , rrdhost_timezone(host) + , rrdhost_tags(host) + , rrdhost_program_name(host) + , rrdhost_program_version(host) , host->rrd_update_every , rrd_memory_mode_name(host->rrd_memory_mode) , host->rrd_history_entries @@ -484,8 +517,8 @@ RRDHOST *rrdhost_create(const char *hostname, , host->cache_dir , host->varlib_dir , host->health_log_filename - , host->health_default_exec - , host->health_default_recipient + , string2str(host->health_default_exec) + , string2str(host->health_default_recipient) ); sql_store_host_system_info(&host->host_uuid, system_info); @@ -537,39 +570,36 @@ void rrdhost_update(RRDHOST *host rrdhost_init_os(host, os); rrdhost_init_timezone(host, timezone, abbrev_timezone, utc_offset); - freez(host->registry_hostname); - host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname); + string_freez(host->registry_hostname); + host->registry_hostname = string_strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname); - if(strcmp(host->hostname, hostname) != 0) { - info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname); - char *t = host->hostname; - host->hostname = strdupz(hostname); - host->hash_hostname = simple_hash(host->hostname); - freez(t); + if(strcmp(rrdhost_hostname(host), hostname) != 0) { + info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", rrdhost_hostname(host), hostname); + rrdhost_init_hostname(host, hostname); } - if(strcmp(host->program_name, program_name) != 0) { - info("Host '%s' switched program name from '%s' to '%s'", host->hostname, host->program_name, program_name); - char *t = host->program_name; - host->program_name = strdupz(program_name); - freez(t); + if(strcmp(rrdhost_program_name(host), program_name) != 0) { + info("Host '%s' switched program name from '%s' to '%s'", rrdhost_hostname(host), rrdhost_program_name(host), program_name); + STRING *t = host->program_name; + host->program_name = string_strdupz(program_name); + string_freez(t); } - if(strcmp(host->program_version, program_version) != 0) { - info("Host '%s' switched program version from '%s' to '%s'", host->hostname, host->program_version, program_version); - char *t = host->program_version; - host->program_version = strdupz(program_version); - freez(t); + if(strcmp(rrdhost_program_version(host), program_version) != 0) { + info("Host '%s' switched program version from '%s' to '%s'", rrdhost_hostname(host), rrdhost_program_version(host), program_version); + STRING *t = host->program_version; + host->program_version = string_strdupz(program_version); + string_freez(t); } if(host->rrd_update_every != update_every) - error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every); + error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", rrdhost_hostname(host), host->rrd_update_every, update_every); if(host->rrd_history_entries < history) - error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history); + error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", rrdhost_hostname(host), host->rrd_history_entries, history); if(host->rrd_memory_mode != mode) - error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); + error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); // update host tags rrdhost_init_tags(host, tags); @@ -591,12 +621,12 @@ void rrdhost_update(RRDHOST *host if (host != localhost) { r = mkdir(host->varlib_dir, 0775); if (r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", host->hostname, host->varlib_dir); + error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir); } snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir); r = mkdir(filename, 0775); if(r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", host->hostname, filename); + error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename); rrdhost_wrlock(host); health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL); @@ -621,7 +651,7 @@ void rrdhost_update(RRDHOST *host rrd_hosts_available++; ml_new_host(host); rrdhost_load_rrdcontext_data(host); - info("Host %s is not in archived mode anymore", host->hostname); + info("Host %s is not in archived mode anymore", rrdhost_hostname(host)); } return; @@ -652,11 +682,11 @@ RRDHOST *rrdhost_find_or_create( debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid); rrd_wrlock(); - RRDHOST *host = rrdhost_find_by_guid(guid, 0); + RRDHOST *host = rrdhost_find_by_guid(guid); if (unlikely(host && RRD_MEMORY_MODE_DBENGINE != mode && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) { /* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */ error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.", - host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); + rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); rrdhost_free(host, 1); host = NULL; } @@ -739,7 +769,7 @@ void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host) { restart_after_removal: rrdhost_foreach_write(host) { if(rrdhost_should_be_removed(host, protected_host, now)) { - info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid); + info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", rrdhost_hostname(host), host->machine_guid); if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) #ifdef ENABLE_DBENGINE @@ -761,6 +791,7 @@ restart_after_removal: // RRDHOST global / startup initialization int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { + rrdhost_init(); if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) @@ -940,19 +971,19 @@ unittest: // there are only used when NETDATA_INTERNAL_CHECKS is set void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line) { - debug(D_RRDHOST, "Checking read lock on host '%s'", host->hostname); + debug(D_RRDHOST, "Checking read lock on host '%s'", rrdhost_hostname(host)); int ret = netdata_rwlock_trywrlock(&host->rrdhost_rwlock); if(ret == 0) - fatal("RRDHOST '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", host->hostname, function, line, file); + fatal("RRDHOST '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", rrdhost_hostname(host), function, line, file); } void __rrdhost_check_wrlock(RRDHOST *host, const char *file, const char *function, const unsigned long line) { - debug(D_RRDHOST, "Checking write lock on host '%s'", host->hostname); + debug(D_RRDHOST, "Checking write lock on host '%s'", rrdhost_hostname(host)); int ret = netdata_rwlock_tryrdlock(&host->rrdhost_rwlock); if(ret == 0) - fatal("RRDHOST '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", host->hostname, function, line, file); + fatal("RRDHOST '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", rrdhost_hostname(host), function, line, file); } void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line) { @@ -1051,7 +1082,7 @@ void rrdhost_free(RRDHOST *host, bool force) { if(!host) return; if (netdata_exit || force) - info("Freeing all memory for host '%s'...", host->hostname); + info("Freeing all memory for host '%s'...", rrdhost_hostname(host)); rrd_check_wrlock(); // make sure the RRDs are write locked @@ -1084,8 +1115,8 @@ void rrdhost_free(RRDHOST *host, bool force) { freez(host->exporting_flags); - while(host->alarms) - rrdcalc_unlink_and_free(host, host->alarms); + while(host->host_alarms) + rrdcalc_unlink_and_free(host, host->host_alarms); RRDCALC *rc,*nc; for(rc = host->alarms_with_foreach; rc ; rc = nc) { @@ -1094,18 +1125,11 @@ void rrdhost_free(RRDHOST *host, bool force) { } host->alarms_with_foreach = NULL; - while(host->templates) - rrdcalctemplate_unlink_and_free(host, host->templates); + while(host->alarms_templates) + rrdcalctemplate_unlink_and_free(host, host->alarms_templates); - RRDCALCTEMPLATE *rt,*next; - for(rt = host->alarms_template_with_foreach; rt ; rt = next) { - next = rt->next; - rrdcalctemplate_free(rt); - } - host->alarms_template_with_foreach = NULL; - - debug(D_RRD_CALLS, "RRDHOST: Cleaning up remaining host variables for host '%s'", host->hostname); - rrdvar_free_remaining_variables(host, &host->rrdvar_root_index); + debug(D_RRD_CALLS, "RRDHOST: Cleaning up remaining host variables for host '%s'", rrdhost_hostname(host)); + rrdvar_free_remaining_variables(host, host->rrdvar_root_index); health_alarm_log_free(host); @@ -1119,7 +1143,7 @@ void rrdhost_free(RRDHOST *host, bool force) { #endif if (!netdata_exit && !force) { - info("Setting archive mode for host '%s'...", host->hostname); + info("Setting archive mode for host '%s'...", rrdhost_hostname(host)); rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); rrdhost_unlock(host); return; @@ -1143,24 +1167,13 @@ void rrdhost_free(RRDHOST *host, bool force) { // ------------------------------------------------------------------------ // remove it from the indexes - if(rrdhost_index_del(host) != host) - error("RRDHOST '%s' removed from index, deleted the wrong entry.", host->hostname); + rrdhost_index_del_hostname(host); + rrdhost_index_del_by_guid(host); // ------------------------------------------------------------------------ // unlink it from the host - if(host == localhost) { - localhost = host->next; - } - else { - // find the previous one - RRDHOST *h; - for(h = localhost; h && h->next != host ; h = h->next) ; - - // bypass it - if(h) h->next = host->next; - else error("Request to free RRDHOST '%s': cannot find it", host->hostname); - } + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(localhost, host, prev, next); // ------------------------------------------------------------------------ // free it @@ -1168,13 +1181,13 @@ void rrdhost_free(RRDHOST *host, bool force) { pthread_mutex_destroy(&host->aclk_state_lock); freez(host->aclk_state.claimed_id); freez(host->aclk_state.prev_claimed_id); - freez((void *)host->tags); + string_freez(host->tags); rrdlabels_destroy(host->host_labels); - freez((void *)host->os); - freez((void *)host->timezone); - freez((void *)host->abbrev_timezone); - freez(host->program_version); - freez(host->program_name); + string_freez(host->os); + string_freez(host->timezone); + string_freez(host->abbrev_timezone); + string_freez(host->program_name); + string_freez(host->program_version); rrdhost_system_info_free(host->system_info); freez(host->cache_dir); freez(host->varlib_dir); @@ -1186,19 +1199,24 @@ void rrdhost_free(RRDHOST *host, bool force) { freez(host->destinations); host->destinations = tmp_destination; } - freez(host->health_default_exec); - freez(host->health_default_recipient); + string_freez(host->health_default_exec); + string_freez(host->health_default_recipient); freez(host->health_log_filename); - freez(host->hostname); - freez(host->registry_hostname); + string_freez(host->registry_hostname); simple_pattern_free(host->rrdpush_send_charts_matching); rrdhost_unlock(host); netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock); netdata_rwlock_destroy(&host->rrdhost_rwlock); freez(host->node_id); + dictionary_destroy(host->rrdset_root_index); + dictionary_destroy(host->rrdset_root_index_name); + dictionary_destroy(host->rrdfamily_root_index); + dictionary_destroy(host->rrdvar_root_index); + rrdhost_destroy_rrdcontexts(host); + string_freez(host->hostname); freez(host); #ifdef ENABLE_ACLK if (wc) @@ -1209,9 +1227,14 @@ void rrdhost_free(RRDHOST *host, bool force) { void rrdhost_free_all(void) { rrd_wrlock(); + /* Make sure child-hosts are released before the localhost. */ - while(localhost->next) rrdhost_free(localhost->next, 1); - rrdhost_free(localhost, 1); + while(localhost && localhost->next) + rrdhost_free(localhost->next, 1); + + if(localhost) + rrdhost_free(localhost, 1); + rrd_unlock(); } @@ -1221,7 +1244,7 @@ void rrdhost_free_all(void) { void rrdhost_save_charts(RRDHOST *host) { if(!host) return; - info("Saving/Closing database of host '%s'...", host->hostname); + info("Saving/Closing database of host '%s'...", rrdhost_hostname(host)); RRDSET *st; @@ -1302,7 +1325,7 @@ static void rrdhost_load_auto_labels(void) { add_aclk_host_labels(); rrdlabels_add( - labels, "_is_parent", (localhost->next || configured_as_parent()) ? "true" : "false", RRDLABEL_SRC_AUTO); + labels, "_is_parent", (rrdhost_hosts_available() > 1 || configured_as_parent()) ? "true" : "false", RRDLABEL_SRC_AUTO); if (localhost->rrdpush_send_destination) rrdlabels_add(labels, "_streams_to", localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO); @@ -1383,7 +1406,7 @@ void reload_host_labels(void) { void rrdhost_delete_charts(RRDHOST *host) { if(!host) return; - info("Deleting database of host '%s'...", host->hostname); + info("Deleting database of host '%s'...", rrdhost_hostname(host)); RRDSET *st; @@ -1408,7 +1431,7 @@ void rrdhost_delete_charts(RRDHOST *host) { void rrdhost_cleanup_charts(RRDHOST *host) { if(!host) return; - info("Cleaning up database of host '%s'...", host->hostname); + info("Cleaning up database of host '%s'...", rrdhost_hostname(host)); RRDSET *st; uint32_t rrdhost_delete_obsolete_charts = rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); @@ -1497,8 +1520,8 @@ restart_after_removal: RRDDIM *rd, *last; rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED); - while (st->variables) rrdsetvar_free(st->variables); - while (st->alarms) rrdsetcalc_unlink(st->alarms); + while (st->variables) rrdsetvar_free(st->variables); + while (st->alarms) rrdsetcalc_unlink(st->alarms); rrdset_wrlock(st); for (rd = st->dimensions, last = NULL ; likely(rd) ; ) { if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { @@ -1555,8 +1578,8 @@ restart_after_removal: } rrdset_unlock(st); - debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", host->hostname, st->id); - rrdvar_free_remaining_variables(host, &st->rrdvar_root_index); + debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", rrdhost_hostname(host), rrdset_id(st)); + rrdvar_free_remaining_variables(host, st->rrdvar_root_index); rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); @@ -1761,46 +1784,6 @@ int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, ch return res; } -/** - * Alarm Compare ID - * - * Callback function used with the binary trees to compare the id of RRDCALC - * - * @param a a pointer to the RRDCAL item to insert,compare or update the binary tree - * @param b the pointer to the binary tree. - * - * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b. - */ -int alarm_compare_id(void *a, void *b) { - register uint32_t hash1 = ((RRDCALC *)a)->id; - register uint32_t hash2 = ((RRDCALC *)b)->id; - - if(hash1 < hash2) return -1; - else if(hash1 > hash2) return 1; - - return 0; -} - -/** - * Alarm Compare NAME - * - * Callback function used with the binary trees to compare the name of RRDCALC - * - * @param a a pointer to the RRDCAL item to insert,compare or update the binary tree - * @param b the pointer to the binary tree. - * - * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b. - */ -int alarm_compare_name(void *a, void *b) { - RRDCALC *in1 = (RRDCALC *)a; - RRDCALC *in2 = (RRDCALC *)b; - - if(in1->hash < in2->hash) return -1; - else if(in1->hash > in2->hash) return 1; - - return strcmp(in1->name,in2->name); -} - // Added for gap-filling, if this proves to be a bottleneck in large-scale systems then we will need to cache // the last entry times as the metric updates, but let's see if it is a problem first. time_t rrdhost_last_entry_t(RRDHOST *h) { diff --git a/database/rrdlabels.c b/database/rrdlabels.c index afd738c3f3..52c88e5083 100644 --- a/database/rrdlabels.c +++ b/database/rrdlabels.c @@ -503,7 +503,7 @@ static void rrdlabel_conflict_callback(const char *name, void *oldvalue, void *n RRDLABEL *lbold = (RRDLABEL *)oldvalue; RRDLABEL *lbnew = (RRDLABEL *)newvalue; - if(lbold->label_value == lbnew->label_value || strcmp(string2str(lbold->label_value), string2str(lbnew->label_value)) == 0) { + if(lbold->label_value == lbnew->label_value) { // they are the same lbold->label_source |= lbnew->label_source; lbold->label_source |= RRDLABEL_FLAG_OLD; @@ -937,9 +937,20 @@ int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, return dictionary_walkthrough_read(labels, label_to_buffer_callback, (void *)&tmp); } +struct label_str { + BUFFER *sql; + int count; + char uuid_str[UUID_STR_LEN]; +}; + static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { - RRDSET *st = (RRDSET *)data; - sql_store_chart_label(st->chart_uuid, (int)ls, (char *)name, (char *)value); + struct label_str *lb = data; + if (unlikely(!lb->count)) + buffer_sprintf(lb->sql, "INSERT OR REPLACE INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "); + else + buffer_strcat(lb->sql, ", "); + buffer_sprintf(lb->sql, "(u2h('%s'), %d,'%s','%s', unixepoch())", lb->uuid_str, ls, name, value); + lb->count++; return 1; } @@ -953,7 +964,12 @@ void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) { rrdcalc_update_rrdlabels(st); // TODO - we should also cleanup sqlite from old new_rrdlabels that have been removed - rrdlabels_walkthrough_read(st->state->chart_labels, chart_label_store_to_sql_callback, st); + BUFFER *sql_buf = buffer_create(1024); + struct label_str tmp = {.sql = sql_buf, .count = 0 }; + uuid_unparse_lower(*st->chart_uuid, tmp.uuid_str); + rrdlabels_walkthrough_read(st->state->chart_labels, chart_label_store_to_sql_callback, &tmp); + db_execute(buffer_tostring(sql_buf)); + buffer_free(sql_buf); } // ---------------------------------------------------------------------------- diff --git a/database/rrdset.c b/database/rrdset.c index aabf303dee..93ea76cc64 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -5,95 +5,85 @@ #include void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line) { - debug(D_RRD_CALLS, "Checking read lock on chart '%s'", st->id); + debug(D_RRD_CALLS, "Checking read lock on chart '%s'", rrdset_id(st)); int ret = netdata_rwlock_trywrlock(&st->rrdset_rwlock); if(ret == 0) - fatal("RRDSET '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", st->id, function, line, file); + fatal("RRDSET '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", rrdset_id(st), function, line, file); } void __rrdset_check_wrlock(RRDSET *st, const char *file, const char *function, const unsigned long line) { - debug(D_RRD_CALLS, "Checking write lock on chart '%s'", st->id); + debug(D_RRD_CALLS, "Checking write lock on chart '%s'", rrdset_id(st)); int ret = netdata_rwlock_tryrdlock(&st->rrdset_rwlock); if(ret == 0) - fatal("RRDSET '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", st->id, function, line, file); + fatal("RRDSET '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", rrdset_id(st), function, line, file); } // ---------------------------------------------------------------------------- // RRDSET index -int rrdset_compare(void* a, void* b) { - if(((RRDSET *)a)->hash < ((RRDSET *)b)->hash) return -1; - else if(((RRDSET *)a)->hash > ((RRDSET *)b)->hash) return 1; - else return strcmp(((RRDSET *)a)->id, ((RRDSET *)b)->id); +static inline void rrdset_index_add(RRDHOST *host, RRDSET *st) { + if(likely(dictionary_set(host->rrdset_root_index, rrdset_id(st), st, sizeof(RRDSET)) == st)) { + rrdset_flag_set(st, RRDSET_FLAG_INDEXED_ID); + } + else { + rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID); + error("RRDSET: %s() attempted to index duplicate object with key '%s'", __FUNCTION__, rrdset_id(st)); + } } -static RRDSET *rrdset_index_find(RRDHOST *host, const char *id, uint32_t hash) { - RRDSET tmp; - strncpyz(tmp.id, id, RRD_ID_LENGTH_MAX); - tmp.hash = (hash)?hash:simple_hash(tmp.id); +static inline void rrdset_index_del(RRDHOST *host, RRDSET *st) { + if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID)) { + if(likely(dictionary_del(host->rrdset_root_index, rrdset_id(st)) == 0)) + rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID); + else + error("RRDSET: %s() attempted to delete non-indexed object with key '%s'", __FUNCTION__, rrdset_id(st)); + } +} - return (RRDSET *)avl_search_lock(&(host->rrdset_root_index), (avl_t *) &tmp); +static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) { + return dictionary_get(host->rrdset_root_index, id); } // ---------------------------------------------------------------------------- // RRDSET name index -#define rrdset_from_avlname(avlname_ptr) ((RRDSET *)((avlname_ptr) - offsetof(RRDSET, avlname))) - -int rrdset_compare_name(void* a, void* b) { - RRDSET *A = rrdset_from_avlname(a); - RRDSET *B = rrdset_from_avlname(b); - - // fprintf(stderr, "COMPARING: %s with %s\n", A->name, B->name); - - if(A->hash_name < B->hash_name) return -1; - else if(A->hash_name > B->hash_name) return 1; - else return strcmp(A->name, B->name); +static inline void rrdset_index_add_name(RRDHOST *host, RRDSET *st) { + if(likely(dictionary_set(host->rrdset_root_index_name, rrdset_name(st), st, sizeof(RRDSET)) == st)) { + rrdset_flag_set(st, RRDSET_FLAG_INDEXED_NAME); + } + else { + rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME); + error("RRDSET: %s() attempted to index duplicate object with key '%s'", __FUNCTION__, rrdset_name(st)); + } } -RRDSET *rrdset_index_add_name(RRDHOST *host, RRDSET *st) { - void *result; - // fprintf(stderr, "ADDING: %s (name: %s)\n", st->id, st->name); - result = avl_insert_lock(&host->rrdset_root_index_name, (avl_t *) (&st->avlname)); - if(result) return rrdset_from_avlname(result); - return NULL; +static inline void rrdset_index_del_name(RRDHOST *host, RRDSET *st) { + if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID)) { + if(likely(dictionary_del(host->rrdset_root_index_name, rrdset_name(st)) != 0)) + rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME); + else + error("RRDSET: %s() attempted to delete non-index object with key '%s'", __FUNCTION__, rrdset_name(st)); + } } -RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st) { - void *result; - // fprintf(stderr, "DELETING: %s (name: %s)\n", st->id, st->name); - result = (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index_name), (avl_t *)(&st->avlname)); - if(result) return rrdset_from_avlname(result); - return NULL; +static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) { + return dictionary_get(host->rrdset_root_index_name, name); } - // ---------------------------------------------------------------------------- // RRDSET - find charts -static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name, uint32_t hash) { - void *result = NULL; - RRDSET tmp; - tmp.name = name; - tmp.hash_name = (hash)?hash:simple_hash(tmp.name); - - result = avl_search_lock(&host->rrdset_root_index_name, (avl_t *) (&(tmp.avlname))); - if(result) return rrdset_from_avlname(result); - - return NULL; -} - inline RRDSET *rrdset_find(RRDHOST *host, const char *id) { - debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, host->hostname); - RRDSET *st = rrdset_index_find(host, id, 0); + debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, rrdhost_hostname(host)); + RRDSET *st = rrdset_index_find(host, id); return(st); } inline RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id) { - debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, host->hostname); + debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, rrdhost_hostname(host)); char buf[RRD_ID_LENGTH_MAX + 1]; strncpyz(buf, type, RRD_ID_LENGTH_MAX - 1); @@ -105,8 +95,8 @@ inline RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *i } inline RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) { - debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, host->hostname); - RRDSET *st = rrdset_index_find_name(host, name, 0); + debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, rrdhost_hostname(host)); + RRDSET *st = rrdset_index_find_name(host, name); return(st); } @@ -129,32 +119,32 @@ char *rrdset_strncpyz_name(char *to, const char *from, size_t length) { } int rrdset_set_name(RRDSET *st, const char *name) { - if(unlikely(st->name && !strcmp(st->name, name))) + if(unlikely(!strcmp(rrdset_name(st), name))) return 1; RRDHOST *host = st->rrdhost; - debug(D_RRD_CALLS, "rrdset_set_name() old: '%s', new: '%s'", st->name?st->name:"", name); + debug(D_RRD_CALLS, "rrdset_set_name() old: '%s', new: '%s'", rrdset_name(st), name); char full_name[RRD_ID_LENGTH_MAX + 1]; char sanitized_name[CONFIG_MAX_VALUE + 1]; char new_name[CONFIG_MAX_VALUE + 1]; - snprintfz(full_name, RRD_ID_LENGTH_MAX, "%s.%s", st->type, name); + snprintfz(full_name, RRD_ID_LENGTH_MAX, "%s.%s", rrdset_type(st), name); rrdset_strncpyz_name(sanitized_name, full_name, CONFIG_MAX_VALUE); strncpyz(new_name, sanitized_name, CONFIG_MAX_VALUE); - if(rrdset_index_find_name(host, new_name, 0)) { - debug(D_RRD_CALLS, "RRDSET: chart name '%s' on host '%s' already exists.", new_name, host->hostname); - if(!strcmp(st->id, full_name) && !st->name) { + if(rrdset_index_find_name(host, new_name)) { + debug(D_RRD_CALLS, "RRDSET: chart name '%s' on host '%s' already exists.", new_name, rrdhost_hostname(host)); + if(!strcmp(rrdset_id(st), full_name) && !st->name) { unsigned i = 1; do { snprintfz(new_name, CONFIG_MAX_VALUE, "%s_%u", sanitized_name, i); i++; - } while (rrdset_index_find_name(host, new_name, 0)); + } while (rrdset_index_find_name(host, new_name)); - info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, host->hostname); + info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, rrdhost_hostname(host)); } else { return 0; } @@ -162,13 +152,12 @@ int rrdset_set_name(RRDSET *st, const char *name) { if(st->name) { rrdset_index_del_name(host, st); - st->name = strdupz(new_name); - st->hash_name = simple_hash(st->name); + string_freez(st->name); + st->name = string_strdupz(new_name); rrdsetvar_rename_all(st); } else { - st->name = strdupz(new_name); - st->hash_name = simple_hash(st->name); + st->name = string_strdupz(new_name); } rrdset_wrlock(st); @@ -177,8 +166,7 @@ int rrdset_set_name(RRDSET *st, const char *name) { rrddimvar_rename_all(rd); rrdset_unlock(st); - if(unlikely(rrdset_index_add_name(host, st) != st)) - error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name); + rrdset_index_add_name(host, st); rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND); rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE); @@ -192,12 +180,13 @@ int rrdset_set_name(RRDSET *st, const char *name) { inline void rrdset_is_obsolete(RRDSET *st) { if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))) { - info("Cannot obsolete already archived chart %s", st->name); + info("Cannot obsolete already archived chart %s", rrdset_name(st)); return; } if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) { rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE); + st->last_accessed_time = now_realtime_sec(); st->rrdhost->obsolete_charts_count++; rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); @@ -212,6 +201,7 @@ inline void rrdset_is_obsolete(RRDSET *st) { inline void rrdset_isnot_obsolete(RRDSET *st) { if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) { rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); + st->last_accessed_time = now_realtime_sec(); st->rrdhost->obsolete_charts_count--; rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); @@ -239,12 +229,12 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) { if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { #ifdef NETDATA_INTERNAL_CHECKS info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").", - rd->name, - st->name, - host->hostname, - rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm), - rd->multiplier, multiplier, - rd->divisor, divisor + rrddim_name(rd), + rrdset_name(st), + rrdhost_hostname(host), + rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm), + rd->multiplier, multiplier, + rd->divisor, divisor ); #endif rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS); @@ -261,7 +251,7 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) { // RRDSET - reset a chart void rrdset_reset(RRDSET *st) { - debug(D_RRD_CALLS, "rrdset_reset() %s", st->name); + debug(D_RRD_CALLS, "rrdset_reset() %s", rrdset_name(st)); st->last_collected_time.tv_sec = 0; st->last_collected_time.tv_usec = 0; @@ -346,9 +336,7 @@ void rrdset_free(RRDSET *st) { // ------------------------------------------------------------------------ // remove it from the indexes - if(unlikely(rrdset_index_del(host, st) != st)) - error("RRDSET: INTERNAL ERROR: attempt to remove from index chart '%s', removed a different chart.", st->id); - + rrdset_index_del(host, st); rrdset_index_del_name(host, st); // ------------------------------------------------------------------------ @@ -361,29 +349,18 @@ void rrdset_free(RRDSET *st) { /* We must free all connected alarms here in case this has been an ephemeral chart whose alarm was * created by a template. This leads to an effective memory leak, which cannot be detected since the * alarms will still be connected to the host, and freed during shutdown. */ - while(st->alarms) rrdcalc_unlink_and_free(st->rrdhost, st->alarms); - while(st->dimensions) rrddim_free(st, st->dimensions); + while(st->alarms) rrdcalc_unlink_and_free(st->rrdhost, st->alarms); + while(st->dimensions) rrddim_free(st, st->dimensions); rrdfamily_free(host, st->rrdfamily); - debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", host->hostname, st->id); - rrdvar_free_remaining_variables(host, &st->rrdvar_root_index); + debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", rrdhost_hostname(host), rrdset_id(st)); + rrdvar_free_remaining_variables(host, st->rrdvar_root_index); // ------------------------------------------------------------------------ // unlink it from the host - if(st == host->rrdset_root) { - host->rrdset_root = st->next; - } - else { - // find the previous one - RRDSET *s; - for(s = host->rrdset_root; s && s->next != st ; s = s->next) ; - - // bypass it - if(s) s->next = st->next; - else error("Request to free RRDSET '%s': cannot find it under host '%s'", st->id, host->hostname); - } + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->rrdset_root, st, prev, next); rrdset_unlock(st); @@ -395,24 +372,28 @@ void rrdset_free(RRDSET *st) { netdata_rwlock_destroy(&st->rrdset_rwlock); - // free directly allocated members - freez((void *)st->name); - freez(st->type); - freez(st->family); - freez(st->title); - freez(st->units); - freez(st->context); - freez(st->cache_dir); - freez(st->plugin_name); - freez(st->module_name); - freez(st->state->old_title); - freez(st->state->old_units); - freez(st->state->old_context); + rrdset_memory_file_free(st); rrdlabels_destroy(st->state->chart_labels); + + // free directly allocated members + + dictionary_destroy(st->rrddim_root_index); + dictionary_destroy(st->rrdvar_root_index); + + string_freez(st->id); + string_freez(st->name); + string_freez(st->type); + string_freez(st->family); + string_freez(st->title); + string_freez(st->units); + string_freez(st->context); + string_freez(st->plugin_name); + string_freez(st->module_name); + + freez(st->cache_dir); freez(st->state); freez(st->chart_uuid); - rrdset_memory_file_free(st); freez(st); } @@ -430,7 +411,7 @@ void rrdset_delete_files(RRDSET *st) { RRDDIM *rd; rrdset_check_rdlock(st); - info("Deleting chart '%s' ('%s') from disk...", st->id, st->name); + info("Deleting chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st)); if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) { const char *cache_filename = rrdset_cache_filename(st); @@ -440,7 +421,7 @@ void rrdset_delete_files(RRDSET *st) { error("Cannot delete chart header file '%s'", cache_filename); } else - error("Cannot find the cache filename of chart '%s'", st->id); + error("Cannot find the cache filename of chart '%s'", rrdset_id(st)); } rrddim_foreach_read(rd, st) { @@ -460,7 +441,7 @@ void rrdset_delete_obsolete_dimensions(RRDSET *st) { rrdset_check_rdlock(st); - info("Deleting dimensions of chart '%s' ('%s') from disk...", st->id, st->name); + info("Deleting dimensions of chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st)); rrddim_foreach_read(rd, st) { if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { @@ -490,9 +471,9 @@ static inline RRDSET *rrdset_find_on_create(RRDHOST *host, const char *fullid) { static inline void rrdset_update_permanent_labels(RRDSET *st) { if(!st->state || !st->state->chart_labels) return; - rrdlabels_add(st->state->chart_labels, "_collect_plugin", st->plugin_name, RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); - rrdlabels_add(st->state->chart_labels, "_collect_module", st->module_name, RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); - rrdlabels_add(st->state->chart_labels, "_instance_family", st->family, RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); + rrdlabels_add(st->state->chart_labels, "_collect_plugin", rrdset_plugin_name(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); + rrdlabels_add(st->state->chart_labels, "_collect_module", rrdset_module_name(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); + rrdlabels_add(st->state->chart_labels, "_instance_family",rrdset_family(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); } RRDSET *rrdset_create_custom( @@ -549,6 +530,7 @@ RRDSET *rrdset_create_custom( char fullid[RRD_ID_LENGTH_MAX + 1]; snprintfz(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id); + json_fix_string(fullid); int changed_from_archived_to_active = 0; RRDSET *st = rrdset_find_on_create(host, fullid); @@ -559,11 +541,9 @@ RRDSET *rrdset_create_custom( changed_from_archived_to_active = 1; mark_rebuild |= META_CHART_ACTIVATED; } - char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_context = NULL, - *old_title_v = NULL, *old_context_v = NULL, *old_units_v = NULL, *old_units = NULL; - int rc; - if(unlikely(name)) + int rc; + if(unlikely(name && *name)) rc = rrdset_set_name(st, name); else rc = rrdset_set_name(st, id); @@ -580,91 +560,61 @@ RRDSET *rrdset_create_custom( mark_rebuild |= META_CHART_UPDATED; } - if (plugin && st->plugin_name) { - if (unlikely(strcmp(plugin, st->plugin_name))) { - old_plugin = st->plugin_name; - st->plugin_name = strdupz(plugin); + if(plugin && *plugin) { + STRING *old_plugin = st->plugin_name; + st->plugin_name = rrd_string_strdupz(plugin); + if (old_plugin != st->plugin_name) mark_rebuild |= META_PLUGIN_UPDATED; - } - } else { - if (plugin != st->plugin_name) { // one is NULL? - old_plugin = st->plugin_name; - st->plugin_name = plugin ? strdupz(plugin) : NULL; - mark_rebuild |= META_PLUGIN_UPDATED; - } + string_freez(old_plugin); } - if (module && st->module_name) { - if (unlikely(strcmp(module, st->module_name))) { - old_module = st->module_name; - st->module_name = strdupz(module); + if(module && *module) { + STRING *old_module = st->module_name; + st->module_name = rrd_string_strdupz(module); + if (old_module != st->module_name) mark_rebuild |= META_MODULE_UPDATED; - } - } else { - if (module != st->module_name) { - if (st->module_name && *st->module_name) { - old_module = st->module_name; - st->module_name = module ? strdupz(module) : NULL; - mark_rebuild |= META_MODULE_UPDATED; - } - } + string_freez(old_module); } - if (unlikely(title && st->state->old_title && strcmp(st->state->old_title, title))) { - char *new_title = strdupz(title); - old_title_v = st->state->old_title; - st->state->old_title = strdupz(title); - json_fix_string(new_title); - old_title = st->title; - st->title = new_title; - mark_rebuild |= META_CHART_UPDATED; + if(title && *title) { + STRING *old_title = st->title; + st->title = rrd_string_strdupz(title); + if(old_title != st->title) + mark_rebuild |= META_CHART_UPDATED; + string_freez(old_title); } - if (unlikely(units && st->state->old_units && strcmp(st->state->old_units, units))) { - char *new_units = strdupz(units); - old_units_v = st->state->old_units; - st->state->old_units = strdupz(units); - json_fix_string(new_units); - old_units= st->units; - st->units = new_units; - mark_rebuild |= META_CHART_UPDATED; + if(units && *units) { + STRING *old_units = st->units; + st->units = rrd_string_strdupz(units); + if(old_units != st->units) + mark_rebuild |= META_CHART_UPDATED; + string_freez(old_units); } + if(context && *context) { + STRING *old_context = st->context; + st->context = rrd_string_strdupz(context); + if(old_context != st->context) + mark_rebuild |= META_CHART_UPDATED; + string_freez(old_context); + } if (st->chart_type != chart_type) { st->chart_type = chart_type; mark_rebuild |= META_CHART_UPDATED; } - if (unlikely(context && st->state->old_context && strcmp(st->state->old_context, context))) { - char *new_context = strdupz(context); - old_context_v = st->state->old_context; - st->state->old_context = strdupz(context); - json_fix_string(new_context); - old_context = st->context; - st->context = new_context; - st->hash_context = simple_hash(st->context); - mark_rebuild |= META_CHART_UPDATED; - } - if (mark_rebuild) { rrdset_flag_clear(st, RRDSET_FLAG_ACLK); - freez(old_plugin); - freez(old_module); - freez(old_title); - freez(old_units); - freez(old_context); - freez(old_title_v); - freez(old_units_v); - freez(old_context_v); if (mark_rebuild != META_CHART_ACTIVATED) { - info("Collector updated metadata for chart %s", st->id); + info("Collector updated metadata for chart %s", rrdset_id(st)); sched_yield(); } } if (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED)) { - debug(D_METADATALOG, "CHART [%s] metadata updated", st->id); - int rc = update_chart_metadata(st->chart_uuid, st, id, name); + debug(D_METADATALOG, "CHART [%s] metadata updated", rrdset_id(st)); + rc = update_chart_metadata(st->chart_uuid, st, id, name); if (unlikely(rc)) error_report("Failed to update chart metadata in the database"); @@ -719,8 +669,7 @@ RRDSET *rrdset_create_custom( st = callocz(1, sizeof(RRDSET)); st->state = callocz(1, sizeof(*st->state)); - strcpy(st->id, fullid); - st->hash = simple_hash(st->id); + st->id = string_strdupz(fullid); // fullid is already json_fix'ed st->rrdhost = host; st->cache_dir = cache_dir; @@ -729,29 +678,23 @@ RRDSET *rrdset_create_custom( if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP) { if(!rrdset_memory_load_or_create_map_save(st, memory_mode)) { - info("Failed to use memory mode %s for chart '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name); + info("Failed to use memory mode %s for chart '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st)); memory_mode = RRD_MEMORY_MODE_RAM; } } st->rrd_memory_mode = memory_mode; - st->plugin_name = plugin?strdupz(plugin):NULL; - st->module_name = module?strdupz(module):NULL; + st->plugin_name = rrd_string_strdupz(plugin); + st->module_name = rrd_string_strdupz(module); st->chart_type = chart_type; - st->type = strdupz(type); - st->family = family ? strdupz(family) : strdupz(st->type); - json_fix_string(st->family); + st->type = rrd_string_strdupz(type); + st->family = family ? rrd_string_strdupz(family) : string_dup(st->type); - st->state->is_ar_chart = strcmp(st->id, ML_ANOMALY_RATES_CHART_ID) == 0; + st->state->is_ar_chart = strcmp(rrdset_id(st), ML_ANOMALY_RATES_CHART_ID) == 0; - st->units = units ? strdupz(units) : strdupz(""); - st->state->old_units = strdupz(st->units); - json_fix_string(st->units); + st->units = rrd_string_strdupz(units); - st->context = context ? strdupz(context) : strdupz(st->id); - st->state->old_context = strdupz(st->context); - json_fix_string(st->context); - st->hash_context = simple_hash(st->context); + st->context = (context && *context) ? rrd_string_strdupz(context) : string_dup(st->id); st->priority = priority; @@ -762,8 +705,17 @@ RRDSET *rrdset_create_custom( st->gap_when_lost_iterations_above = (int) (gap_when_lost_iterations_above + 2); - avl_init_lock(&st->dimensions_index, rrddim_compare); - avl_init_lock(&st->rrdvar_root_index, rrdvar_compare); + st->rrddim_root_index = dictionary_create( + DICTIONARY_FLAG_NAME_LINK_DONT_CLONE + | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE + | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE + ); + + st->rrdvar_root_index = dictionary_create( + DICTIONARY_FLAG_NAME_LINK_DONT_CLONE + | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE + | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE + ); netdata_rwlock_init(&st->rrdset_rwlock); st->state->chart_labels = rrdlabels_create(); @@ -776,14 +728,11 @@ RRDSET *rrdset_create_custom( // could not use the name, use the id rrdset_set_name(st, id); - st->title = strdupz(title); - st->state->old_title = strdupz(st->title); - json_fix_string(st->title); + st->title = rrd_string_strdupz(title); - st->rrdfamily = rrdfamily_create(host, st->family); + st->rrdfamily = rrdfamily_create(host, rrdset_family(st)); - st->next = host->rrdset_root; - host->rrdset_root = st; + DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->rrdset_root, st, prev, next); if(host->health_enabled) { rrdsetvar_create(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); @@ -793,9 +742,7 @@ RRDSET *rrdset_create_custom( rrdsetvar_create(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, RRDVAR_OPTION_DEFAULT); } - if(unlikely(rrdset_index_add(host, st) != st)) - error("RRDSET: INTERNAL ERROR: attempt to index duplicate chart '%s'", st->id); - + rrdset_index_add(host, st); rrdsetcalc_link_matching(st); rrdcalctemplate_link_matching(st); @@ -870,7 +817,7 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) { // oops! the database is in the future #ifdef NETDATA_INTERNAL_CHECKS info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER - " secs in the future (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC, st->counter, st->counter_done); + " secs in the future (counter #%zu, update #%zu). Adjusting it to current time.", rrdset_id(st), rrdhost_hostname(st->rrdhost), (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC, st->counter, st->counter_done); #endif st->last_collected_time.tv_sec = now.tv_sec - st->update_every; @@ -890,7 +837,7 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) { // oops! the database is too far behind #ifdef NETDATA_INTERNAL_CHECKS info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER - " secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done); + " secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", rrdset_id(st), rrdhost_hostname(st->rrdhost), (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done); #endif microseconds = (usec_t)since_last_usec; @@ -925,11 +872,11 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) { } #ifdef NETDATA_INTERNAL_CHECKS - debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds); + debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", rrdset_name(st), microseconds); rrdset_debug(st, "NEXT: %llu microseconds", microseconds); if(discarded && discarded != microseconds) - info("host '%s', chart '%s': discarded data collection time of %llu usec, replaced with %llu usec, reason: '%s'", st->rrdhost->hostname, st->id, discarded, microseconds, discard_reason?discard_reason:"UNDEFINED"); + info("host '%s', chart '%s': discarded data collection time of %llu usec, replaced with %llu usec, reason: '%s'", rrdhost_hostname(st->rrdhost), rrdset_id(st), discarded, microseconds, discard_reason?discard_reason:"UNDEFINED"); #endif @@ -1081,6 +1028,8 @@ static void store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, t->last_collected_ut = point_end_time_ut; store_metric_at_tier(rd, t, sp, point_end_time_ut); } + + rrdcontext_collected_rrddim(rd); } static inline size_t rrdset_done_interpolate( @@ -1114,7 +1063,7 @@ static inline size_t rrdset_done_interpolate( for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) { #ifdef NETDATA_INTERNAL_CHECKS - if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); } + if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", rrdset_name(st), first_ut, last_stored_ut, next_store_ut, now_collect_ut); } rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC); rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC); #endif @@ -1140,7 +1089,7 @@ static inline size_t rrdset_done_interpolate( NETDATA_DOUBLE_FORMAT " * (%llu - %llu)" " / (%llu - %llu)" - , rd->name + , rrddim_name(rd) , new_value , rd->calculated_value , next_store_ut, last_collect_ut @@ -1157,7 +1106,7 @@ static inline size_t rrdset_done_interpolate( #ifdef NETDATA_INTERNAL_CHECKS rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " NETDATA_DOUBLE_FORMAT " - EXTRAPOLATING", - rd->name + rrddim_name(rd) , (NETDATA_DOUBLE)(next_store_ut - last_stored_ut) ); #endif @@ -1193,7 +1142,7 @@ static inline size_t rrdset_done_interpolate( rrdset_debug(st, "%s: CALC2 DEF " NETDATA_DOUBLE_FORMAT " = (((" "(" NETDATA_DOUBLE_FORMAT " - " NETDATA_DOUBLE_FORMAT ")" " * %llu" - " / %llu) + " NETDATA_DOUBLE_FORMAT, rd->name + " / %llu) + " NETDATA_DOUBLE_FORMAT, rrddim_name(rd) , new_value , rd->calculated_value, rd->last_calculated_value , (next_store_ut - first_ut) @@ -1225,7 +1174,7 @@ static inline size_t rrdset_done_interpolate( (void) ml_is_anomalous(rd, 0, false); #ifdef NETDATA_INTERNAL_CHECKS - rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rd->name, current_entry); + rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rrddim_name(rd), current_entry); #endif store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE); @@ -1275,7 +1224,7 @@ static inline void rrdset_done_fill_the_gap(RRDSET *st) { current_entry = ((current_entry + 1) >= entries) ? 0 : current_entry + 1; #ifdef NETDATA_INTERNAL_CHECKS - rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING (FILLED THE GAP)", rd->name, current_entry); + rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING (FILLED THE GAP)", rrddim_name(rd), current_entry); #endif } } @@ -1294,8 +1243,7 @@ static inline void rrdset_done_fill_the_gap(RRDSET *st) { void rrdset_done(RRDSET *st) { if(unlikely(netdata_exit)) return; - debug(D_RRD_CALLS, "rrdset_done() for chart %s", st->name); - rrdcontext_collected_rrdset(st); + debug(D_RRD_CALLS, "rrdset_done() for chart %s", rrdset_name(st)); RRDDIM *rd; @@ -1326,7 +1274,7 @@ void rrdset_done(RRDSET *st) { #endif if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) { - error("Chart '%s' has the OBSOLETE flag set, but it is collected.", st->id); + error("Chart '%s' has the OBSOLETE flag set, but it is collected.", rrdset_id(st)); rrdset_isnot_obsolete(st); } @@ -1334,7 +1282,7 @@ void rrdset_done(RRDSET *st) { if(unlikely(st->usec_since_last_update > st->entries * update_every_ut && st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && st->rrd_memory_mode != RRD_MEMORY_MODE_NONE)) { info("host '%s', chart %s: took too long to be updated (counter #%zu, update #%zu, %0.3" NETDATA_DOUBLE_MODIFIER - " secs). Resetting it.", st->rrdhost->hostname, st->name, st->counter, st->counter_done, (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC); + " secs). Resetting it.", rrdhost_hostname(st->rrdhost), rrdset_name(st), st->counter, st->counter_done, (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC); rrdset_reset(st); st->usec_since_last_update = update_every_ut; store_this_entry = 0; @@ -1382,7 +1330,7 @@ void rrdset_done(RRDSET *st) { info( "%s: too old data (last updated at %"PRId64".%"PRId64", last collected at %"PRId64".%"PRId64"). " "Resetting it. Will not store the next entry.", - st->name, + rrdset_name(st), (int64_t)st->last_updated.tv_sec, (int64_t)st->last_updated.tv_usec, (int64_t)st->last_collected_time.tv_sec, @@ -1404,7 +1352,7 @@ void rrdset_done(RRDSET *st) { info( "%s: too old data (last updated at %" PRId64 ".%" PRId64 ", last collected at %" PRId64 ".%" PRId64 "). " "Resetting it. Will not store the next entry.", - st->name, + rrdset_name(st), (int64_t)st->last_updated.tv_sec, (int64_t)st->last_updated.tv_usec, (int64_t)st->last_collected_time.tv_sec, @@ -1467,9 +1415,9 @@ after_first_database_work: if(unlikely(st->rrdhost->rrdpush_send_enabled)) rrdset_done_push(st); - if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE)) { + + if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE)) goto after_second_database_work; - } #ifdef NETDATA_INTERNAL_CHECKS rrdset_debug(st, "last_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last collection time)", (NETDATA_DOUBLE)last_collect_ut/USEC_PER_SEC); @@ -1506,7 +1454,7 @@ after_first_database_work: } if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) { - error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rd->name, st->id); + error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st)); rrddim_isnot_obsolete(st, rd); } @@ -1515,11 +1463,12 @@ after_first_database_work: " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " NETDATA_DOUBLE_FORMAT - " calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name - , rd->last_collected_value - , rd->collected_value - , rd->last_calculated_value - , rd->calculated_value + " calculated_value = " NETDATA_DOUBLE_FORMAT + , rrddim_name(rd) + , rd->last_collected_value + , rd->collected_value + , rd->last_calculated_value + , rd->calculated_value ); #endif @@ -1533,7 +1482,7 @@ after_first_database_work: rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN " NETDATA_DOUBLE_FORMAT " = " COLLECTED_NUMBER_FORMAT " * " NETDATA_DOUBLE_FORMAT - " / " NETDATA_DOUBLE_FORMAT, rd->name + " / " NETDATA_DOUBLE_FORMAT, rrddim_name(rd) , rd->calculated_value , rd->collected_value , (NETDATA_DOUBLE)rd->multiplier @@ -1558,7 +1507,7 @@ after_first_database_work: rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100" " * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT - , rd->name + , rrddim_name(rd) , rd->calculated_value , rd->collected_value , st->collected_total @@ -1579,7 +1528,7 @@ after_first_database_work: // produces wrong results as far as incremental counters are concerned. if(unlikely((uint64_t)rd->last_collected_value > (uint64_t)rd->collected_value)) { debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT - , st->name, rd->name + , rrdset_name(st), rrddim_name(rd) , rd->last_collected_value , rd->collected_value); @@ -1627,7 +1576,7 @@ after_first_database_work: COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" " * " NETDATA_DOUBLE_FORMAT - " / " NETDATA_DOUBLE_FORMAT, rd->name + " / " NETDATA_DOUBLE_FORMAT, rrddim_name(rd) , rd->calculated_value , rd->collected_value, rd->last_collected_value , (NETDATA_DOUBLE)rd->multiplier @@ -1647,7 +1596,7 @@ after_first_database_work: // to reset the calculation (it will give zero as the calculation for this second) if(unlikely(rd->last_collected_value > rd->collected_value)) { debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT - , st->name, rd->name + , rrdset_name(st), rrddim_name(rd) , rd->last_collected_value , rd->collected_value ); @@ -1672,7 +1621,7 @@ after_first_database_work: rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100" " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" - , rd->name + , rrddim_name(rd) , rd->calculated_value , rd->collected_value, rd->last_collected_value , st->collected_total, st->last_collected_total @@ -1688,7 +1637,7 @@ after_first_database_work: #ifdef NETDATA_INTERNAL_CHECKS rrdset_debug(st, "%s: CALC " NETDATA_DOUBLE_FORMAT " = 0" - , rd->name + , rrddim_name(rd) , rd->calculated_value ); #endif @@ -1701,7 +1650,7 @@ after_first_database_work: " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " NETDATA_DOUBLE_FORMAT - " calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name + " calculated_value = " NETDATA_DOUBLE_FORMAT, rrddim_name(rd) , rd->last_collected_value , rd->collected_value , rd->last_calculated_value @@ -1718,7 +1667,7 @@ after_first_database_work: // if(unlikely(now_collect_ut < next_store_ut && st->counter_done > 1)) { // // this is collected in the same interpolation point // rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT"); -// info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, st->name, st->counter_done, next_store_ut - now_collect_ut); +// info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, rrdset_name(st), st->counter_done, next_store_ut - now_collect_ut); // } // #endif @@ -1738,6 +1687,7 @@ after_second_database_work: #ifdef ENABLE_ACLK time_t mark = now_realtime_sec(); #endif + rrddim_foreach_read(rd, st) { if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) continue; @@ -1752,7 +1702,7 @@ after_second_database_work: continue; #ifdef NETDATA_INTERNAL_CHECKS - rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rd->name, rd->last_collected_value, rd->collected_value); + rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rrddim_name(rd), rd->last_collected_value, rd->collected_value); #endif rd->last_collected_value = rd->collected_value; @@ -1762,7 +1712,7 @@ after_second_database_work: if(unlikely(!first_entry)) { #ifdef NETDATA_INTERNAL_CHECKS rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT - ") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value); + ") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rrddim_name(rd), rd->last_calculated_value + rd->calculated_value, rd->calculated_value); #endif rd->last_calculated_value += rd->calculated_value; @@ -1779,7 +1729,7 @@ after_second_database_work: case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: #ifdef NETDATA_INTERNAL_CHECKS rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT - ") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value); + ") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rrddim_name(rd), rd->last_calculated_value, rd->calculated_value); #endif rd->last_calculated_value = rd->calculated_value; @@ -1795,7 +1745,7 @@ after_second_database_work: " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " NETDATA_DOUBLE_FORMAT - " calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name + " calculated_value = " NETDATA_DOUBLE_FORMAT, rrddim_name(rd) , rd->last_collected_value , rd->collected_value , rd->last_calculated_value @@ -1835,7 +1785,7 @@ after_second_database_work: for( rd = st->dimensions, last = NULL ; likely(rd) ; ) { if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && !rrddim_flag_check(rd, RRDDIM_FLAG_ACLK) && (rd->last_collected_time.tv_sec + rrdset_free_obsolete_time < now))) { - info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id); + info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rrddim_name(rd), rrddim_id(rd), rrdset_name(st), rrdset_id(st)); const char *cache_filename = rrddim_cache_filename(rd); if(cache_filename) { @@ -1900,8 +1850,9 @@ after_second_database_work: } } - rrdset_unlock(st); + rrdcontext_collected_rrdset(st); + rrdset_unlock(st); netdata_thread_enable_cancelability(); } @@ -2047,8 +1998,8 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo info("Initializing file '%s'.", fullfilename); memset(st_on_file, 0, size); } - else if(strncmp(st_on_file->id, st->id, RRD_ID_LENGTH_MAX_V019) != 0) { - error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, st->id); + else if(strncmp(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019) != 0) { + error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, rrdset_id(st)); memset(st_on_file, 0, size); } else if(st_on_file->memsize != size || st_on_file->entries != st->entries) { @@ -2094,7 +2045,7 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo memset(st_on_file, 0, size); // set the values we need - strncpyz(st_on_file->id, st->id, RRD_ID_LENGTH_MAX_V019 + 1); + strncpyz(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019 + 1); strcpy(st_on_file->cache_filename, fullfilename); strcpy(st_on_file->magic, RRDSET_MAGIC_V019); st_on_file->memsize = size; diff --git a/database/rrdsetvar.c b/database/rrdsetvar.c index e520764a2f..7234cd94ee 100644 --- a/database/rrdsetvar.c +++ b/database/rrdsetvar.c @@ -13,31 +13,31 @@ static inline void rrdsetvar_free_variables(RRDSETVAR *rs) { // ------------------------------------------------------------------------ // CHART - rrdvar_free(host, &st->rrdvar_root_index, rs->var_local); + rrdvar_free(host, st->rrdvar_root_index, rs->var_local); rs->var_local = NULL; // ------------------------------------------------------------------------ // FAMILY - rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family); + rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family); rs->var_family = NULL; - rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_name); + rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_name); rs->var_family_name = NULL; // ------------------------------------------------------------------------ // HOST - rrdvar_free(host, &host->rrdvar_root_index, rs->var_host); + rrdvar_free(host, host->rrdvar_root_index, rs->var_host); rs->var_host = NULL; - rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_name); + rrdvar_free(host, host->rrdvar_root_index, rs->var_host_name); rs->var_host_name = NULL; // ------------------------------------------------------------------------ // KEYS - freez(rs->key_fullid); + string_freez(rs->key_fullid); rs->key_fullid = NULL; - freez(rs->key_fullname); + string_freez(rs->key_fullname); rs->key_fullname = NULL; } @@ -58,40 +58,38 @@ static inline void rrdsetvar_create_variables(RRDSETVAR *rs) { // KEYS char buffer[RRDVAR_MAX_LENGTH + 1]; - snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rs->variable); - rs->key_fullid = strdupz(buffer); + snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), string2str(rs->variable)); + rs->key_fullid = string_strdupz(buffer); - snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable); - rs->key_fullname = strdupz(buffer); + snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), string2str(rs->variable)); + rs->key_fullname = string_strdupz(buffer); // ------------------------------------------------------------------------ // CHART - rs->var_local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->variable, rs->type, options, rs->value); + rs->var_local = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->variable, rs->type, options, rs->value); // ------------------------------------------------------------------------ // FAMILY - rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value); - rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value); + rs->var_family = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value); + rs->var_family_name = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value); // ------------------------------------------------------------------------ // HOST - rs->var_host = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value); - rs->var_host_name = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value); + rs->var_host = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value); + rs->var_host_name = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value); } RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options) { - debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable); + debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", rrdset_id(st), rrdset_name(st), variable); RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR)); - rs->variable = strdupz(variable); - rs->hash = simple_hash(rs->variable); + rs->variable = string_strdupz(variable); rs->type = type; rs->value = value; rs->options = options; rs->rrdset = st; - rs->next = st->variables; - st->variables = rs; + DOUBLE_LINKED_LIST_PREPEND_UNSAFE(st->variables, rs, prev, next); rrdsetvar_create_variables(rs); @@ -99,7 +97,7 @@ RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, } void rrdsetvar_rename_all(RRDSET *st) { - debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name); + debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", rrdset_id(st), rrdset_name(st)); RRDSETVAR *rs; for(rs = st->variables; rs ; rs = rs->next) @@ -110,21 +108,13 @@ void rrdsetvar_rename_all(RRDSET *st) { void rrdsetvar_free(RRDSETVAR *rs) { RRDSET *st = rs->rrdset; - debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable); + debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", rrdset_id(st), rrdset_name(st), string2str(rs->variable)); - if(st->variables == rs) { - st->variables = rs->next; - } - else { - RRDSETVAR *t; - for (t = st->variables; t && t->next != rs; t = t->next); - if(!t) error("RRDSETVAR '%s' not found in chart '%s' variables linked list", rs->key_fullname, st->id); - else t->next = rs->next; - } + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->variables, rs, prev, next); rrdsetvar_free_variables(rs); - freez(rs->variable); + string_freez(rs->variable); if(rs->options & RRDVAR_OPTION_ALLOCATED) freez(rs->value); @@ -138,24 +128,22 @@ void rrdsetvar_free(RRDSETVAR *rs) { RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name) { RRDHOST *host = st->rrdhost; - char *n = strdupz(name); - rrdvar_fix_name(n); - uint32_t hash = simple_hash(n); + STRING *n = rrdvar_name_to_string(name); rrdset_wrlock(st); // find it RRDSETVAR *rs; for(rs = st->variables; rs ; rs = rs->next) { - if(hash == rs->hash && strcmp(n, rs->variable) == 0) { + if(rs->variable == n) { rrdset_unlock(st); if(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) { - freez(n); + string_freez(n); return rs; } else { - error("RRDSETVAR: custom variable '%s' on chart '%s' of host '%s', conflicts with an internal chart variable", n, st->id, host->hostname); - freez(n); + error("RRDSETVAR: custom variable '%s' on chart '%s' of host '%s', conflicts with an internal chart variable", string2str(n), rrdset_id(st), rrdhost_hostname(host)); + string_freez(n); return NULL; } } @@ -166,17 +154,17 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name) NETDATA_DOUBLE *v = mallocz(sizeof(NETDATA_DOUBLE)); *v = NAN; - rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR); + rs = rrdsetvar_create(st, string2str(n), RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR); rrdset_unlock(st); - freez(n); + string_freez(n); return rs; } void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, NETDATA_DOUBLE value) { if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) || !(rs->options & RRDVAR_OPTION_ALLOCATED)) { error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " NETDATA_DOUBLE_FORMAT - " but the variable is not a custom chart one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value); + " but the variable is not a custom chart one.", string2str(rs->variable), rrdset_id(rs->rrdset), rrdhost_hostname(rs->rrdset->rrdhost), value); } else { NETDATA_DOUBLE *v = rs->value; diff --git a/database/rrdsetvar.h b/database/rrdsetvar.h index 37f4da9591..bcfa27aa63 100644 --- a/database/rrdsetvar.h +++ b/database/rrdsetvar.h @@ -12,11 +12,10 @@ // these variables struct rrdsetvar { - char *variable; // variable name - uint32_t hash; // variable name hash + STRING *variable; // variable name - char *key_fullid; // chart type.chart id.variable - char *key_fullname; // chart type.chart name.variable + STRING *key_fullid; // chart type.chart id.variable + STRING *key_fullname; // chart type.chart name.variable RRDVAR_TYPE type; void *value; @@ -32,6 +31,7 @@ struct rrdsetvar { struct rrdset *rrdset; struct rrdsetvar *next; + struct rrdsetvar *prev; }; extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name); diff --git a/database/rrdvar.c b/database/rrdvar.c index d4dda10799..acf8cdd8b1 100644 --- a/database/rrdvar.c +++ b/database/rrdvar.c @@ -20,87 +20,74 @@ inline int rrdvar_fix_name(char *variable) { return fixed; } -int rrdvar_compare(void* a, void* b) { - if(((RRDVAR *)a)->hash < ((RRDVAR *)b)->hash) return -1; - else if(((RRDVAR *)a)->hash > ((RRDVAR *)b)->hash) return 1; - else return strcmp(((RRDVAR *)a)->name, ((RRDVAR *)b)->name); +static inline RRDVAR *rrdvar_index_add(DICTIONARY *dict, RRDVAR *rv) { + return dictionary_set(dict, rrdvar_name(rv), rv, sizeof(RRDVAR)); } -static inline RRDVAR *rrdvar_index_add(avl_tree_lock *tree, RRDVAR *rv) { - RRDVAR *ret = (RRDVAR *)avl_insert_lock(tree, (avl_t *)(rv)); - if(ret != rv) - debug(D_VARIABLES, "Request to insert RRDVAR '%s' into index failed. Already exists.", rv->name); +static inline RRDVAR *rrdvar_index_del(DICTIONARY *dict, RRDVAR *rv) { + if(dictionary_del(dict, rrdvar_name(rv)) != 0) { + error("Request to remove RRDVAR '%s' from index failed. Not Found.", rrdvar_name(rv)); + return NULL; + } - return ret; + return rv; } -static inline RRDVAR *rrdvar_index_del(avl_tree_lock *tree, RRDVAR *rv) { - RRDVAR *ret = (RRDVAR *)avl_remove_lock(tree, (avl_t *)(rv)); - if(!ret) - error("Request to remove RRDVAR '%s' from index failed. Not Found.", rv->name); - - return ret; +static inline RRDVAR *rrdvar_index_find(DICTIONARY *dict, STRING *name) { + return dictionary_get(dict, string2str(name)); } -static inline RRDVAR *rrdvar_index_find(avl_tree_lock *tree, const char *name, uint32_t hash) { - RRDVAR tmp; - tmp.name = (char *)name; - tmp.hash = (hash)?hash:simple_hash(tmp.name); - - return (RRDVAR *)avl_search_lock(tree, (avl_t *)&tmp); -} - -inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) { +inline void rrdvar_free(RRDHOST *host, DICTIONARY *dict, RRDVAR *rv) { (void)host; if(!rv) return; - if(tree) { - debug(D_VARIABLES, "Deleting variable '%s'", rv->name); - if(unlikely(!rrdvar_index_del(tree, rv))) - error("RRDVAR: Attempted to delete variable '%s' from host '%s', but it is not found.", rv->name, host->hostname); + if(dict) { + debug(D_VARIABLES, "Deleting variable '%s'", rrdvar_name(rv)); + if(unlikely(!rrdvar_index_del(dict, rv))) + error("RRDVAR: Attempted to delete variable '%s' from host '%s', but it is not found.", rrdvar_name(rv), rrdhost_hostname(host)); } if(rv->options & RRDVAR_OPTION_ALLOCATED) freez(rv->value); - freez(rv->name); + string_freez(rv->name); freez(rv); } -inline RRDVAR *rrdvar_create_and_index(const char *scope __maybe_unused, avl_tree_lock *tree, const char *name, - RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value) { +inline STRING *rrdvar_name_to_string(const char *name) { char *variable = strdupz(name); rrdvar_fix_name(variable); - uint32_t hash = simple_hash(variable); + STRING *name_string = string_strdupz(variable); + freez(variable); + return name_string; +} - RRDVAR *rv = rrdvar_index_find(tree, variable, hash); +inline RRDVAR *rrdvar_create_and_index(const char *scope __maybe_unused, DICTIONARY *dict, STRING *name, + RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value) { + + RRDVAR *rv = rrdvar_index_find(dict, name); if(unlikely(!rv)) { - debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", variable, scope); + debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", string2str(name), scope); rv = callocz(1, sizeof(RRDVAR)); - rv->name = variable; - rv->hash = hash; + rv->name = string_dup(name); rv->type = type; rv->options = options; rv->value = value; rv->last_updated = now_realtime_sec(); - RRDVAR *ret = rrdvar_index_add(tree, rv); + RRDVAR *ret = rrdvar_index_add(dict, rv); if(unlikely(ret != rv)) { - debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", variable, scope); + debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", string2str(name), scope); freez(rv); - freez(variable); rv = NULL; } else - debug(D_VARIABLES, "Variable '%s' created in scope '%s'", variable, scope); + debug(D_VARIABLES, "Variable '%s' created in scope '%s'", string2str(name), scope); } else { - debug(D_VARIABLES, "Variable '%s' is already found in scope '%s'.", variable, scope); - - // already exists - freez(variable); + debug(D_VARIABLES, "Variable '%s' is already found in scope '%s'.", string2str(name), scope); // this is important // it must return NULL - not the existing variable - or double-free will happen @@ -110,57 +97,48 @@ inline RRDVAR *rrdvar_create_and_index(const char *scope __maybe_unused, avl_tre return rv; } -void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock) { - // This is not bullet proof - avl should support some means to destroy it - // with a callback for each item already in the index - - RRDVAR *rv, *last = NULL; - while((rv = (RRDVAR *)tree_lock->avl_tree.root)) { - if(unlikely(rv == last)) { - error("RRDVAR: INTERNAL ERROR: Cannot cleanup tree of RRDVARs"); - break; - } - last = rv; - rrdvar_free(host, tree_lock, rv); +void rrdvar_free_remaining_variables(RRDHOST *host, DICTIONARY *dict) { + RRDVAR *rv; + dfe_start_rw(dict, rv, DICTIONARY_LOCK_REENTRANT) { + rrdvar_free(host, dict, rv); } + dfe_done(rv); } // ---------------------------------------------------------------------------- // CUSTOM HOST VARIABLES -inline int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void * /*rrdvar*/, void * /*data*/), void *data) { - return avl_traverse_lock(&host->rrdvar_root_index, callback, data); +inline int rrdvar_walkthrough_read(DICTIONARY *dict, int (*callback)(const char *name, void *rrdvar, void *data), void *data) { + return dictionary_walkthrough_read(dict, callback, data); } -static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *tree_lock, const char *name) { +static RRDVAR *rrdvar_custom_variable_create(const char *scope, DICTIONARY *dict, const char *name) { NETDATA_DOUBLE *v = callocz(1, sizeof(NETDATA_DOUBLE)); *v = NAN; - RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v); + STRING *name_string = rrdvar_name_to_string(name); + + RRDVAR *rv = rrdvar_create_and_index(scope, dict, name_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v); if(unlikely(!rv)) { freez(v); - debug(D_VARIABLES, "Requested variable '%s' already exists - possibly 2 plugins are updating it at the same time.", name); - - char *variable = strdupz(name); - rrdvar_fix_name(variable); - uint32_t hash = simple_hash(variable); + debug(D_VARIABLES, "Requested variable '%s' already exists - possibly 2 plugins are updating it at the same time.", string2str(name_string)); // find the existing one to return it - rv = rrdvar_index_find(tree_lock, variable, hash); - - freez(variable); + rv = rrdvar_index_find(dict, name_string); } + string_freez(name_string); + return rv; } RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) { - return rrdvar_custom_variable_create("host", &host->rrdvar_root_index, name); + return rrdvar_custom_variable_create("host", host->rrdvar_root_index, name); } void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value) { if(rv->type != RRDVAR_TYPE_CALCULATED || !(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR) || !(rv->options & RRDVAR_OPTION_ALLOCATED)) - error("requested to set variable '%s' to value " NETDATA_DOUBLE_FORMAT " but the variable is not a custom one.", rv->name, value); + error("requested to set variable '%s' to value " NETDATA_DOUBLE_FORMAT " but the variable is not a custom one.", rrdvar_name(rv), value); else { NETDATA_DOUBLE *v = rv->value; if(*v != value) { @@ -174,10 +152,6 @@ void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE v } } -int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR * /*rv*/, void * /*data*/), void *data) { - return avl_traverse_lock(&host->rrdvar_root_index, (int (*)(void *, void *))callback, data); -} - // ---------------------------------------------------------------------------- // RRDVAR lookup @@ -214,26 +188,26 @@ NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) { } } -int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, NETDATA_DOUBLE *result) { +int health_variable_lookup(STRING *variable, RRDCALC *rc, NETDATA_DOUBLE *result) { RRDSET *st = rc->rrdset; if(!st) return 0; RRDHOST *host = st->rrdhost; RRDVAR *rv; - rv = rrdvar_index_find(&st->rrdvar_root_index, variable, hash); + rv = rrdvar_index_find(st->rrdvar_root_index, variable); if(rv) { *result = rrdvar2number(rv); return 1; } - rv = rrdvar_index_find(&st->rrdfamily->rrdvar_root_index, variable, hash); + rv = rrdvar_index_find(st->rrdfamily->rrdvar_root_index, variable); if(rv) { *result = rrdvar2number(rv); return 1; } - rv = rrdvar_index_find(&host->rrdvar_root_index, variable, hash); + rv = rrdvar_index_find(host->rrdvar_root_index, variable); if(rv) { *result = rrdvar2number(rv); return 1; @@ -251,16 +225,16 @@ struct variable2json_helper { RRDVAR_OPTIONS options; }; -static int single_variable2json(void *entry, void *data) { +static int single_variable2json(const char *name __maybe_unused, void *entry, void *data) { struct variable2json_helper *helper = (struct variable2json_helper *)data; RRDVAR *rv = (RRDVAR *)entry; NETDATA_DOUBLE value = rrdvar2number(rv); if (helper->options == RRDVAR_OPTION_DEFAULT || rv->options & helper->options) { if(unlikely(isnan(value) || isinf(value))) - buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rv->name); + buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rrdvar_name(rv)); else - buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" NETDATA_DOUBLE_MODIFIER, helper->counter?",":"", rv->name, (NETDATA_DOUBLE)value); + buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" NETDATA_DOUBLE_MODIFIER, helper->counter?",":"", rrdvar_name(rv), (NETDATA_DOUBLE)value); helper->counter++; } @@ -276,7 +250,7 @@ void health_api_v1_chart_custom_variables2json(RRDSET *st, BUFFER *buf) { }; buffer_sprintf(buf, "{"); - avl_traverse_lock(&st->rrdvar_root_index, single_variable2json, (void *)&helper); + rrdvar_walkthrough_read(st->rrdvar_root_index, single_variable2json, &helper); buffer_strcat(buf, "\n\t\t\t}"); } @@ -289,16 +263,16 @@ void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf) { .options = RRDVAR_OPTION_DEFAULT }; - buffer_sprintf(buf, "{\n\t\"chart\": \"%s\",\n\t\"chart_name\": \"%s\",\n\t\"chart_context\": \"%s\",\n\t\"chart_variables\": {", st->id, st->name, st->context); - avl_traverse_lock(&st->rrdvar_root_index, single_variable2json, (void *)&helper); + buffer_sprintf(buf, "{\n\t\"chart\": \"%s\",\n\t\"chart_name\": \"%s\",\n\t\"chart_context\": \"%s\",\n\t\"chart_variables\": {", rrdset_id(st), rrdset_name(st), rrdset_context(st)); + rrdvar_walkthrough_read(st->rrdvar_root_index, single_variable2json, &helper); - buffer_sprintf(buf, "\n\t},\n\t\"family\": \"%s\",\n\t\"family_variables\": {", st->family); + buffer_sprintf(buf, "\n\t},\n\t\"family\": \"%s\",\n\t\"family_variables\": {", rrdset_family(st)); helper.counter = 0; - avl_traverse_lock(&st->rrdfamily->rrdvar_root_index, single_variable2json, (void *)&helper); + rrdvar_walkthrough_read(st->rrdfamily->rrdvar_root_index, single_variable2json, &helper); - buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",\n\t\"host_variables\": {", host->hostname); + buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",\n\t\"host_variables\": {", rrdhost_hostname(host)); helper.counter = 0; - avl_traverse_lock(&host->rrdvar_root_index, single_variable2json, (void *)&helper); + rrdvar_walkthrough_read(host->rrdvar_root_index, single_variable2json, &helper); buffer_strcat(buf, "\n\t}\n}\n"); } diff --git a/database/rrdvar.h b/database/rrdvar.h index 9074edcdb1..4ea367dbb7 100644 --- a/database/rrdvar.h +++ b/database/rrdvar.h @@ -5,8 +5,6 @@ #include "libnetdata/libnetdata.h" -extern int rrdvar_compare(void *a, void *b); - typedef enum rrdvar_type { RRDVAR_TYPE_CALCULATED = 1, RRDVAR_TYPE_TIME_T = 2, @@ -32,35 +30,34 @@ typedef enum rrdvar_options { // 2. at each context (RRDFAMILY.rrdvar_root_index) // 3. at each host (RRDHOST.rrdvar_root_index) struct rrdvar { - avl_t avl; - - char *name; - uint32_t hash; - - RRDVAR_TYPE type; - RRDVAR_OPTIONS options; + STRING *name; void *value; - time_t last_updated; + + RRDVAR_OPTIONS options:16; + RRDVAR_TYPE type:8; }; +#define rrdvar_name(rv) string2str((rv)->name) + #define RRDVAR_MAX_LENGTH 1024 extern int rrdvar_fix_name(char *variable); #include "rrd.h" +extern STRING *rrdvar_name_to_string(const char *name); + extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name); extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value); -extern int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data); -extern void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock); +extern void rrdvar_free_remaining_variables(RRDHOST *host, DICTIONARY *dict); -extern int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data); +extern int rrdvar_walkthrough_read(DICTIONARY *dict, int (*callback)(const char *name, void *rrdvar, void *data), void *data); extern NETDATA_DOUBLE rrdvar2number(RRDVAR *rv); -extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value); -extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv); +extern RRDVAR *rrdvar_create_and_index(const char *scope, DICTIONARY *dict, STRING *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value); +extern void rrdvar_free(RRDHOST *host, DICTIONARY *dict, RRDVAR *rv); #endif //NETDATA_RRDVAR_H diff --git a/database/sqlite/sqlite_aclk.c b/database/sqlite/sqlite_aclk.c index b51ea317d3..1312aa5712 100644 --- a/database/sqlite/sqlite_aclk.c +++ b/database/sqlite/sqlite_aclk.c @@ -237,15 +237,14 @@ struct aclk_database_worker_config *find_inactive_wc_by_node_id(char *node_id) void aclk_sync_exit_all() { rrd_rdlock(); - RRDHOST *host = localhost; - while(host) { + RRDHOST *host; + rrdhost_foreach_read(host) { struct aclk_database_worker_config *wc = host->dbsync_worker; if (wc) { wc->is_shutting_down = 1; (void) aclk_database_deq_cmd(wc); uv_cond_signal(&wc->cmd_cond); } - host = host->next; } rrd_unlock(); @@ -320,7 +319,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column char node_str[UUID_STR_LEN] = ""; if (likely(host->node_id)) uuid_unparse_lower(*host->node_id, node_str); - internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", host->hostname, host->machine_guid, node_str); + internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", rrdhost_hostname(host), host->machine_guid, node_str); #endif return 0; } @@ -335,7 +334,7 @@ int aclk_start_sync_thread(void *data, int argc, char **argv, char **column) uuid_unparse_lower(*((uuid_t *) argv[0]), uuid_str); - RRDHOST *host = rrdhost_find_by_guid(uuid_str, 0); + RRDHOST *host = rrdhost_find_by_guid(uuid_str); if (host == localhost) return 0; @@ -526,7 +525,7 @@ void aclk_database_worker(void *arg) char threadname[NETDATA_THREAD_NAME_MAX+1]; if (wc->host) - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->host->hostname); + snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", rrdhost_hostname(wc->host)); else { snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->uuid_str); threadname[11] = '\0'; @@ -705,14 +704,14 @@ void aclk_database_worker(void *arg) case ACLK_DATABASE_TIMER: if (unlikely(localhost && !wc->host && !wc->is_orphan)) { if (claimed()) { - wc->host = rrdhost_find_by_guid(wc->host_guid, 0); + wc->host = rrdhost_find_by_guid(wc->host_guid); if (wc->host) { - info("HOST %s (%s) detected as active", wc->host->hostname, wc->host_guid); - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->host->hostname); + info("HOST %s (%s) detected as active", rrdhost_hostname(wc->host), wc->host_guid); + snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", rrdhost_hostname(wc->host)); uv_thread_set_name_np(wc->thread, threadname); wc->host->dbsync_worker = wc; if (unlikely(!wc->hostname)) - wc->hostname = strdupz(wc->host->hostname); + wc->hostname = strdupz(rrdhost_hostname(wc->host)); aclk_del_worker_thread(wc); wc->node_info_send = 1; } @@ -844,7 +843,7 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id) uuid_unparse_lower(*node_id, wc->node_id); if (likely(host)) { host->dbsync_worker = (void *)wc; - wc->hostname = strdupz(host->hostname); + wc->hostname = strdupz(rrdhost_hostname(host)); } else wc->hostname = get_hostname_by_node_id(wc->node_id); @@ -1039,12 +1038,11 @@ void aclk_data_rotated(void) time_t next_rotation_time = now_realtime_sec()+ACLK_DATABASE_ROTATION_DELAY; rrd_rdlock(); - RRDHOST *this_host = localhost; - while (this_host) { + RRDHOST *this_host; + rrdhost_foreach_read(this_host) { struct aclk_database_worker_config *wc = this_host->dbsync_worker; if (wc) wc->rotation_after = next_rotation_time; - this_host = this_host->next; } rrd_unlock(); diff --git a/database/sqlite/sqlite_aclk.h b/database/sqlite/sqlite_aclk.h index b73f422e16..3560f0d3d6 100644 --- a/database/sqlite/sqlite_aclk.h +++ b/database/sqlite/sqlite_aclk.h @@ -216,13 +216,17 @@ static inline RRDHOST *find_host_by_node_id(char *node_id) if (uuid_parse(node_id, node_uuid)) return NULL; - RRDHOST *host = localhost; - while(host) { - if (host->node_id && !(uuid_compare(*host->node_id, node_uuid))) - return host; - host = host->next; + rrd_rdlock(); + RRDHOST *host, *ret = NULL; + rrdhost_foreach_read(host) { + if (host->node_id && !(uuid_compare(*host->node_id, node_uuid))) { + ret = host; + break; + } } - return NULL; + rrd_unlock(); + + return ret; } diff --git a/database/sqlite/sqlite_aclk_alert.c b/database/sqlite/sqlite_aclk_alert.c index c95b98f263..3e6f3aaa40 100644 --- a/database/sqlite/sqlite_aclk_alert.c +++ b/database/sqlite/sqlite_aclk_alert.c @@ -220,7 +220,7 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d int rc; if (unlikely(!wc->alert_updates)) { - log_access("ACLK STA [%s (%s)]: Ignoring alert push event, updates have been turned off for this node.", wc->node_id, wc->host ? wc->host->hostname : "N/A"); + log_access("ACLK STA [%s (%s)]: Ignoring alert push event, updates have been turned off for this node.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A"); return; } @@ -300,9 +300,9 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d alarm_log.config_hash = strdupz((char *)uuid_str); alarm_log.utc_offset = wc->host->utc_offset; - alarm_log.timezone = strdupz((char *)wc->host->abbrev_timezone); + alarm_log.timezone = strdupz(rrdhost_abbrev_timezone(wc->host)); alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) : - strdupz((char *)wc->host->health_default_exec); + strdupz((char *)string2str(wc->host->health_default_exec)); alarm_log.conf_source = strdupz((char *)sqlite3_column_text(res, 16)); char *edit_command = sqlite3_column_bytes(res, 16) > 0 ? @@ -374,7 +374,7 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d log_access( "ACLK RES [%s (%s)]: ALERTS SENT from %" PRIu64 " to %" PRIu64 " batch=%" PRIu64, wc->node_id, - wc->host ? wc->host->hostname : "N/A", + wc->host ? rrdhost_hostname(wc->host) : "N/A", log_first_sequence_id, log_last_sequence_id, wc->alerts_batch_id); @@ -424,9 +424,7 @@ void aclk_send_alarm_health_log(char *node_id) struct aclk_database_worker_config *wc = find_inactive_wc_by_node_id(node_id); if (likely(!wc)) { - rrd_rdlock(); RRDHOST *host = find_host_by_node_id(node_id); - rrd_unlock(); if (likely(host)) wc = (struct aclk_database_worker_config *)host->dbsync_worker; } @@ -460,9 +458,7 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a RRDHOST *host = wc->host; if (unlikely(!host)) { - rrd_rdlock(); host = find_host_by_node_id(wc->node_id); - rrd_unlock(); if (unlikely(!host)) { log_access( @@ -554,7 +550,7 @@ void aclk_send_alarm_configuration(char *config_hash) return; } - log_access("ACLK REQ [%s (%s)]: Request to send alert config %s.", wc->node_id, wc->host ? wc->host->hostname : "N/A", config_hash); + log_access("ACLK REQ [%s (%s)]: Request to send alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash); struct aclk_database_cmd cmd; memset(&cmd, 0, sizeof(cmd)); @@ -664,14 +660,14 @@ int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct } if (likely(p_alarm_config.cfg_hash)) { - log_access("ACLK RES [%s (%s)]: Sent alert config %s.", wc->node_id, wc->host ? wc->host->hostname : "N/A", config_hash); + log_access("ACLK RES [%s (%s)]: Sent alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash); aclk_send_provide_alarm_cfg(&p_alarm_config); freez((char *) cmd.data_param); freez(p_alarm_config.cfg_hash); destroy_aclk_alarm_configuration(&alarm_config); } else - log_access("ACLK STA [%s (%s)]: Alert config for %s not found.", wc->node_id, wc->host ? wc->host->hostname : "N/A", config_hash); + log_access("ACLK STA [%s (%s)]: Alert config for %s not found.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash); bind_fail: rc = sqlite3_finalize(res); @@ -697,9 +693,7 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start return; struct aclk_database_worker_config *wc = NULL; - rrd_rdlock(); RRDHOST *host = find_host_by_node_id(node_id); - rrd_unlock(); if (likely(host)) { wc = (struct aclk_database_worker_config *)host->dbsync_worker ? (struct aclk_database_worker_config *)host->dbsync_worker : @@ -716,7 +710,7 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start wc = (struct aclk_database_worker_config *)find_inactive_wc_by_node_id(node_id); if (likely(wc)) { - log_access("ACLK REQ [%s (%s)]: ALERTS STREAM from %"PRIu64" batch=%"PRIu64, node_id, wc->host ? wc->host->hostname : "N/A", start_seq_id, batch_id); + log_access("ACLK REQ [%s (%s)]: ALERTS STREAM from %"PRIu64" batch=%"PRIu64, node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", start_seq_id, batch_id); __sync_synchronize(); wc->alerts_batch_id = batch_id; wc->alerts_start_seq_id = start_seq_id; @@ -744,7 +738,7 @@ void sql_process_queue_removed_alerts_to_aclk(struct aclk_database_worker_config db_execute(buffer_tostring(sql)); - log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS", wc->node_id, wc->host ? wc->host->hostname : "N/A"); + log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A"); buffer_free(sql); @@ -780,17 +774,15 @@ void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, uint64_t sn return; struct aclk_database_worker_config *wc = NULL; - rrd_rdlock(); RRDHOST *host = find_host_by_node_id(node_id); if (likely(host)) wc = (struct aclk_database_worker_config *)host->dbsync_worker; - rrd_unlock(); if (likely(wc)) { log_access( "IN [%s (%s)]: Request to send alerts snapshot, snapshot_id %" PRIu64 " and ack_sequence_id %" PRIu64, wc->node_id, - wc->host ? wc->host->hostname : "N/A", + wc->host ? rrdhost_hostname(wc->host) : "N/A", snapshot_id, sequence_id); if (wc->alerts_snapshot_id == snapshot_id) @@ -831,13 +823,13 @@ void aclk_mark_alert_cloud_ack(char *uuid_str, uint64_t alerts_ack_sequence_id) #ifdef ENABLE_ACLK void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_ENTRY *ae, RRDHOST *host) { - char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN"); + char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN"); char config_hash_id[GUID_LEN + 1]; uuid_unparse_lower(ae->config_hash_id, config_hash_id); - alarm_log->chart = strdupz((char *)ae->chart); - alarm_log->name = strdupz((char *)ae->name); - alarm_log->family = strdupz((char *)ae->family); + alarm_log->chart = strdupz(ae_chart_name(ae)); + alarm_log->name = strdupz(ae_name(ae)); + alarm_log->family = strdupz(ae_family(ae)); alarm_log->batch_id = 0; alarm_log->sequence_id = 0; @@ -846,9 +838,9 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN alarm_log->config_hash = strdupz((char *)config_hash_id); alarm_log->utc_offset = host->utc_offset; - alarm_log->timezone = strdupz((char *)host->abbrev_timezone); - alarm_log->exec_path = ae->exec ? strdupz((char *)ae->exec) : strdupz((char *)host->health_default_exec); - alarm_log->conf_source = ae->source ? strdupz((char *)ae->source) : strdupz((char *)""); + alarm_log->timezone = strdupz(rrdhost_abbrev_timezone(host)); + alarm_log->exec_path = ae->exec ? strdupz(ae_exec(ae)) : strdupz((char *)string2str(host->health_default_exec)); + alarm_log->conf_source = ae->source ? strdupz(ae_source(ae)) : strdupz((char *)""); alarm_log->command = strdupz((char *)edit_command); @@ -861,19 +853,19 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN alarm_log->last_repeat = (time_t)ae->last_repeat; alarm_log->silenced = - ((ae->flags & HEALTH_ENTRY_FLAG_SILENCED) || (ae->recipient && !strncmp((char *)ae->recipient, "silent", 6))) ? + ((ae->flags & HEALTH_ENTRY_FLAG_SILENCED) || (ae->recipient && !strncmp(ae_recipient(ae), "silent", 6))) ? 1 : 0; - alarm_log->value_string = strdupz(ae->new_value_string); - alarm_log->old_value_string = strdupz(ae->old_value_string); + alarm_log->value_string = strdupz(ae_new_value_string(ae)); + alarm_log->old_value_string = strdupz(ae_old_value_string(ae)); alarm_log->value = (!isnan(ae->new_value)) ? (NETDATA_DOUBLE)ae->new_value : 0; alarm_log->old_value = (!isnan(ae->old_value)) ? (NETDATA_DOUBLE)ae->old_value : 0; alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0; - alarm_log->rendered_info = ae->info ? strdupz(ae->info) : strdupz((char *)""); - alarm_log->chart_context = ae->chart_context ? strdupz(ae->chart_context) : strdupz((char *)""); + alarm_log->rendered_info = strdupz(ae_info(ae)); + alarm_log->chart_context = strdupz(ae_chart_context(ae)); freez(edit_command); } @@ -905,7 +897,7 @@ void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, stru UNUSED(cmd); // we perhaps we don't need this for snapshots if (unlikely(!wc->alert_updates)) { - log_access("ACLK STA [%s (%s)]: Ignoring alert snapshot event, updates have been turned off for this node.", wc->node_id, wc->host ? wc->host->hostname : "N/A"); + log_access("ACLK STA [%s (%s)]: Ignoring alert snapshot event, updates have been turned off for this node.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A"); return; } @@ -921,7 +913,7 @@ void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, stru if (unlikely(!claim_id)) return; - log_access("ACLK REQ [%s (%s)]: Sending alerts snapshot, snapshot_id %" PRIu64, wc->node_id, wc->host ? wc->host->hostname : "N/A", wc->alerts_snapshot_id); + log_access("ACLK REQ [%s (%s)]: Sending alerts snapshot, snapshot_id %" PRIu64, wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", wc->alerts_snapshot_id); aclk_mark_alert_cloud_ack(wc->uuid_str, wc->alerts_ack_sequence_id); diff --git a/database/sqlite/sqlite_aclk_chart.c b/database/sqlite/sqlite_aclk_chart.c index 0d9b9cda06..8e7bb6ba07 100644 --- a/database/sqlite/sqlite_aclk_chart.c +++ b/database/sqlite/sqlite_aclk_chart.c @@ -153,10 +153,10 @@ int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_dat chart_payload.config_hash = get_str_from_uuid(&st->state->hash_id); chart_payload.update_every = st->update_every; chart_payload.memory_mode = st->rrd_memory_mode; - chart_payload.name = (char *)st->name; + chart_payload.name = (char *)rrdset_name(st); chart_payload.node_id = wc->node_id; chart_payload.claim_id = claim_id; - chart_payload.id = strdupz(st->id); + chart_payload.id = strdupz(rrdset_id(st)); chart_payload.chart_labels = rrdlabels_create(); rrdlabels_copy(chart_payload.chart_labels, st->state->chart_labels); @@ -304,7 +304,7 @@ void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_d log_access( "ACLK STA [%s (%s)]: Ignoring chart push event, updates have been turned off for this node.", wc->node_id, - wc->host ? wc->host->hostname : "N/A"); + wc->host ? rrdhost_hostname(wc->host) : "N/A"); return; } @@ -504,7 +504,7 @@ int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_d log_access( "ACLK REQ [%s (%s)]: Sending chart config for %s.", wc->node_id, - wc->host ? wc->host->hostname : "N/A", + wc->host ? rrdhost_hostname(wc->host) : "N/A", hash_id); aclk_chart_config_updated(&chart_config, 1); destroy_chart_config_updated(&chart_config); @@ -512,7 +512,7 @@ int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_d log_access( "ACLK STA [%s (%s)]: Chart config for %s not found.", wc->node_id, - wc->host ? wc->host->hostname : "N/A", + wc->host ? rrdhost_hostname(wc->host) : "N/A", hash_id); bind_fail: @@ -552,7 +552,7 @@ void aclk_receive_chart_ack(struct aclk_database_worker_config *wc, struct aclk_ log_access( "ACLK STA [%s (%s)]: CHARTS ACKNOWLEDGED IN THE DATABASE UP TO %" PRIu64, wc->node_id, - wc->host ? wc->host->hostname : "N/A", + wc->host ? rrdhost_hostname(wc->host) : "N/A", cmd.param1); bind_fail: @@ -637,7 +637,7 @@ void aclk_get_chart_config(char **hash_id) log_access( "ACLK REQ [%s (%s)]: Request %d for chart config with hash %s received.", wc->node_id, - wc->host ? wc->host->hostname : "N/A", + wc->host ? rrdhost_hostname(wc->host) : "N/A", i, hash_id[i]); cmd.data_param = (void *)strdupz(hash_id[i]); @@ -660,11 +660,10 @@ static void aclk_submit_param_command(char *node_id, enum aclk_database_opcode a cmd.opcode = aclk_command; cmd.param1 = param; - rrd_rdlock(); RRDHOST *host = find_host_by_node_id(node_id); if (likely(host)) wc = (struct aclk_database_worker_config *)host->dbsync_worker; - rrd_unlock(); + if (wc) aclk_database_enq_cmd(wc, &cmd); else { @@ -702,8 +701,8 @@ void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at struct aclk_database_worker_config *wc = find_inactive_wc_by_node_id(node_id); rrd_rdlock(); - RRDHOST *host = localhost; - while(host) { + RRDHOST *host; + rrdhost_foreach_read(host) { if (wc || (host->node_id && !(uuid_compare(*host->node_id, node_uuid)))) { rrd_unlock(); if (!wc) @@ -771,10 +770,8 @@ void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at } return; } - host = host->next; } rrd_unlock(); - return; } #define SQL_SELECT_HOST_MEMORY_MODE "SELECT memory_mode FROM chart WHERE host_id = @host_id LIMIT 1;" @@ -826,7 +823,7 @@ void aclk_update_retention(struct aclk_database_worker_config *wc) return; if (wc->host && rrdhost_flag_check(wc->host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) { - internal_error(true, "Skipping aclk_update_retention for host %s because context streaming is enabled", wc->host->hostname); + internal_error(true, "Skipping aclk_update_retention for host %s because context streaming is enabled", rrdhost_hostname(wc->host)); return; } @@ -1093,9 +1090,9 @@ void queue_dimension_to_aclk(RRDDIM *rd, time_t last_updated) memset(&dim_payload, 0, sizeof(dim_payload)); dim_payload.node_id = wc->node_id; dim_payload.claim_id = claim_id; - dim_payload.name = rd->name; - dim_payload.id = rd->id; - dim_payload.chart_id = rd->rrdset->id; + dim_payload.name = rrddim_name(rd); + dim_payload.id = rrddim_id(rd); + dim_payload.chart_id = rrdset_id(rd->rrdset); dim_payload.created_at.tv_sec = created_at; dim_payload.last_timestamp.tv_sec = last_updated; @@ -1144,9 +1141,9 @@ void aclk_send_dimension_update(RRDDIM *rd) rd->rrdset->rrdhost->dbsync_worker, claim_id, &rd->metric_uuid, - rd->id, - rd->name, - rd->rrdset->id, + rrddim_id(rd), + rrddim_name(rd), + rrdset_id(rd->rrdset), first_entry_t, live ? 0 : last_entry_t, NULL); @@ -1155,9 +1152,9 @@ void aclk_send_dimension_update(RRDDIM *rd) debug( D_ACLK_SYNC, "%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld)", - rd->rrdset->rrdhost->hostname, - rd->rrdset->name, - rd->name, + rrdhost_hostname(rd->rrdset->rrdhost), + rrdset_name(rd->rrdset), + rrddim_name(rd), live, first_entry_t, last_entry_t); @@ -1165,9 +1162,9 @@ void aclk_send_dimension_update(RRDDIM *rd) debug( D_ACLK_SYNC, "%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld) collected %ld seconds ago", - rd->rrdset->rrdhost->hostname, - rd->rrdset->name, - rd->name, + rrdhost_hostname(rd->rrdset->rrdhost), + rrdset_name(rd->rrdset), + rrddim_name(rd), live, first_entry_t, last_entry_t, @@ -1280,15 +1277,15 @@ void sql_check_chart_liveness(RRDSET *st) { if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) { if (likely(st->dimensions && st->counter_done && !queue_chart_to_aclk(st))) { - debug(D_ACLK_SYNC,"Check chart liveness [%s] submit chart definition", st->name); + debug(D_ACLK_SYNC,"Check chart liveness [%s] submit chart definition", rrdset_name(st)); rrdset_flag_set(st, RRDSET_FLAG_ACLK); } } else - debug(D_ACLK_SYNC,"Check chart liveness [%s] chart definition already submitted", st->name); + debug(D_ACLK_SYNC,"Check chart liveness [%s] chart definition already submitted", rrdset_name(st)); time_t mark = now_realtime_sec(); - debug(D_ACLK_SYNC,"Check chart liveness [%s] scanning dimensions", st->name); + debug(D_ACLK_SYNC,"Check chart liveness [%s] scanning dimensions", rrdset_name(st)); rrddim_foreach_read(rd, st) { if (!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, mark)); diff --git a/database/sqlite/sqlite_aclk_node.c b/database/sqlite/sqlite_aclk_node.c index 3d11f83aa8..246d5d0536 100644 --- a/database/sqlite/sqlite_aclk_node.c +++ b/database/sqlite/sqlite_aclk_node.c @@ -16,8 +16,8 @@ DICTIONARY *collectors_from_charts(RRDHOST *host, DICTIONARY *dict) { rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { struct collector_info col = { - .plugin = st->plugin_name ? st->plugin_name : "", - .module = st->module_name ? st->module_name : "" + .plugin = rrdset_plugin_name(st), + .module = rrdset_module_name(st) }; snprintfz(name, 499, "%s:%s", col.plugin, col.module); dictionary_set(dict, name, &col, sizeof(struct collector_info)); @@ -47,7 +47,7 @@ void sql_build_node_collectors(struct aclk_database_worker_config *wc) dictionary_destroy(dict); freez(upd_node_collectors.claim_id); - log_access("ACLK RES [%s (%s)]: NODE COLLECTORS SENT", wc->node_id, wc->host->hostname); + log_access("ACLK RES [%s (%s)]: NODE COLLECTORS SENT", wc->node_id, rrdhost_hostname(wc->host)); #else UNUSED(wc); #endif @@ -94,8 +94,8 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat netdata_mutex_unlock(&host->receiver_lock); } - node_info.data.name = host->hostname; - node_info.data.os = (char *) host->os; + node_info.data.name = rrdhost_hostname(host); + node_info.data.os = rrdhost_os(host); node_info.data.os_name = host->system_info->host_os_name; node_info.data.os_version = host->system_info->host_os_version; node_info.data.kernel_name = host->system_info->kernel_name; @@ -106,8 +106,8 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat node_info.data.memory = host->system_info->host_ram_total ? host->system_info->host_ram_total : "0"; node_info.data.disk_space = host->system_info->host_disk_space ? host->system_info->host_disk_space : "0"; node_info.data.version = host_version ? host_version : VERSION; - node_info.data.release_channel = (char *) get_release_channel(); - node_info.data.timezone = (char *) host->abbrev_timezone; + node_info.data.release_channel = get_release_channel(); + node_info.data.timezone = rrdhost_abbrev_timezone(host); node_info.data.virtualization_type = host->system_info->virtualization ? host->system_info->virtualization : "unknown"; node_info.data.container_type = host->system_info->container ? host->system_info->container : "unknown"; node_info.data.custom_info = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", ""); @@ -126,7 +126,7 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat node_info.data.host_labels_ptr = host->host_labels; aclk_update_node_info(&node_info); - log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, wc->host->hostname, wc->host_guid, wc->host == localhost ? "parent" : "child"); + log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, rrdhost_hostname(wc->host), wc->host_guid, wc->host == localhost ? "parent" : "child"); rrd_unlock(); freez(node_info.claim_id); diff --git a/database/sqlite/sqlite_context.c b/database/sqlite/sqlite_context.c index cf3810af8b..94c5919245 100644 --- a/database/sqlite/sqlite_context.c +++ b/database/sqlite/sqlite_context.c @@ -21,7 +21,6 @@ const char *database_context_cleanup[] = { }; sqlite3 *db_context_meta = NULL; - /* * Initialize the SQLite database * Return 0 on success diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index e0e9b26d37..c0724ba9e3 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -29,6 +29,7 @@ const char *database_config[] = { "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int);", "CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);", "CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);", + "CREATE INDEX IF NOT EXISTS ind_c2 on chart (host_id, context);", "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, " "label_value text, date_created int, PRIMARY KEY (chart_id, label_key));", "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created);", @@ -408,6 +409,24 @@ int init_database_batch(sqlite3 *database, int rebuild, int init_type, const cha return 0; } +static void _uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + uuid_t uuid; + + if ( argc != 1 ){ + sqlite3_result_null(context); + return ; + } + int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid); + if (rc == -1) { + sqlite3_result_null(context); + return ; + } + + sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); +} + + /* * Initialize the SQLite database * Return 0 on success @@ -527,6 +546,9 @@ int sql_init_database(db_check_action_type_t rebuild, int memory) for (int i = 0; i < MAX_PREPARED_STATEMENTS; i++) (void)pthread_key_create(&key_pool[i], release_statement); + rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, _uuid_parse, 0, 0); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to register internal u2h function"); return 0; } @@ -605,11 +627,11 @@ int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid) if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = sqlite3_bind_text(res, 2, rd->id, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 2, rrddim_id(rd), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = sqlite3_bind_text(res, 3, rd->name, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 3, rrddim_name(rd), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; @@ -620,7 +642,7 @@ int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid) } else { uuid_generate(*store_uuid); - status = sql_store_dimension(store_uuid, st->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor, rd->algorithm); + status = sql_store_dimension(store_uuid, st->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); if (unlikely(status)) error_report("Failed to store dimension metadata in the database"); } @@ -745,8 +767,11 @@ int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const return 0; rc = sql_store_chart( - chart_uuid, &st->rrdhost->host_uuid, st->type, id, name, st->family, st->context, st->title, st->units, st->plugin_name, - st->module_name, st->priority, st->update_every, st->chart_type, st->rrd_memory_mode, st->entries); + chart_uuid, &st->rrdhost->host_uuid, rrdset_type(st), id, name, + rrdset_family(st), rrdset_context(st), rrdset_title(st), rrdset_units(st), + rrdset_plugin_name(st), rrdset_module_name(st), + st->priority, st->update_every, st->chart_type, + st->rrd_memory_mode, st->entries); return rc; } @@ -762,7 +787,7 @@ uuid_t *create_chart_uuid(RRDSET *st, const char *id, const char *name) #ifdef NETDATA_INTERNAL_CHECKS char uuid_str[GUID_LEN + 1]; uuid_unparse_lower(*uuid, uuid_str); - debug(D_METADATALOG,"Generating uuid [%s] for chart %s under host %s", uuid_str, st->id, st->rrdhost->hostname); + debug(D_METADATALOG,"Generating uuid [%s] for chart %s under host %s", uuid_str, rrdset_id(st), rrdhost_hostname(st->rrdhost)); #endif rc = update_chart_metadata(uuid, st, id, name); @@ -922,11 +947,11 @@ int sql_store_host_info(RRDHOST *host) if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = bind_text_null(res, 2, host->hostname, 0); + rc = bind_text_null(res, 2, rrdhost_hostname(host), 0); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = bind_text_null(res, 3, host->registry_hostname, 1); + rc = bind_text_null(res, 3, rrdhost_registry_hostname(host), 1); if (unlikely(rc != SQLITE_OK)) goto bind_fail; @@ -934,15 +959,15 @@ int sql_store_host_info(RRDHOST *host) if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = bind_text_null(res, 5, host->os, 1); + rc = bind_text_null(res, 5, rrdhost_os(host), 1); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = bind_text_null(res, 6, host->timezone, 1); + rc = bind_text_null(res, 6, rrdhost_timezone(host), 1); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = bind_text_null(res, 7, host->tags, 1); + rc = bind_text_null(res, 7, rrdhost_tags(host), 1); if (unlikely(rc != SQLITE_OK)) goto bind_fail; @@ -954,7 +979,7 @@ int sql_store_host_info(RRDHOST *host) if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = bind_text_null(res, 10, host->abbrev_timezone, 1); + rc = bind_text_null(res, 10, rrdhost_abbrev_timezone(host), 1); if (unlikely(rc != SQLITE_OK)) goto bind_fail; @@ -962,11 +987,11 @@ int sql_store_host_info(RRDHOST *host) if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = bind_text_null(res, 12, host->program_name, 1); + rc = bind_text_null(res, 12, rrdhost_program_name(host), 1); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = bind_text_null(res, 13, host->program_version, 1); + rc = bind_text_null(res, 13, rrdhost_program_version(host), 1); if (unlikely(rc != SQLITE_OK)) goto bind_fail; @@ -980,18 +1005,18 @@ int sql_store_host_info(RRDHOST *host) int store_rc = sqlite3_step_monitored(res); if (unlikely(store_rc != SQLITE_DONE)) - error_report("Failed to store host %s, rc = %d", host->hostname, rc); + error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), rc); rc = sqlite3_reset(res); if (unlikely(rc != SQLITE_OK)) - error_report("Failed to reset statement to store host %s, rc = %d", host->hostname, rc); + error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc); return !(store_rc == SQLITE_DONE); bind_fail: - error_report("Failed to bind parameter to store host %s, rc = %d", host->hostname, rc); + error_report("Failed to bind parameter to store host %s, rc = %d", rrdhost_hostname(host), rc); rc = sqlite3_reset(res); if (unlikely(rc != SQLITE_OK)) - error_report("Failed to reset statement to store host %s, rc = %d", host->hostname, rc); + error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc); return 1; } @@ -1315,11 +1340,11 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb) ",\n\t\"memory_mode\": \"%s\"" ",\n\t\"custom_info\": \"%s\"" ",\n\t\"charts\": {" - , host->hostname - , host->program_version + , rrdhost_hostname(host) + , rrdhost_program_version(host) , get_release_channel() - , host->os - , host->timezone + , rrdhost_os(host) + , rrdhost_timezone(host) , host->rrd_update_every , host->rrd_history_entries , rrd_memory_mode_name(host->rrd_memory_mode) @@ -1410,7 +1435,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb) "\n\t\t\t\"hostname\": \"%s\"" "\n\t\t}" , (found > 0) ? "," : "" - , h->hostname + , rrdhost_hostname(h) ); found++; @@ -1424,7 +1449,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb) , "\n\t\t{" "\n\t\t\t\"hostname\": \"%s\"" "\n\t\t}" - , host->hostname + , rrdhost_hostname(host) ); } @@ -1445,13 +1470,13 @@ failed: void free_temporary_host(RRDHOST *host) { if (host) { - freez(host->hostname); - freez((char *)host->os); - freez((char *)host->tags); - freez((char *)host->timezone); - freez(host->program_name); - freez(host->program_version); - freez(host->registry_hostname); + string_freez(host->hostname); + string_freez(host->os); + string_freez(host->tags); + string_freez(host->timezone); + string_freez(host->program_name); + string_freez(host->program_version); + string_freez(host->registry_hostname); freez(host->system_info); freez(host); } @@ -1506,10 +1531,13 @@ RRDHOST *sql_create_host_by_uuid(char *hostname) host = callocz(1, sizeof(RRDHOST)); - set_host_properties(host, sqlite3_column_int(res, 2), RRD_MEMORY_MODE_DBENGINE, hostname, - (char *) sqlite3_column_text(res, 1), (const char *) uuid_str, - (char *) sqlite3_column_text(res, 3), (char *) sqlite3_column_text(res, 5), - (char *) sqlite3_column_text(res, 4), NULL, 0, NULL, NULL); + set_host_properties(host, sqlite3_column_int(res, 2), + RRD_MEMORY_MODE_DBENGINE, + (char *) sqlite3_column_text(res, 1), + (char *) sqlite3_column_text(res, 3), + (char *) sqlite3_column_text(res, 5), + (char *) sqlite3_column_text(res, 4), + NULL, 0, NULL, NULL); uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0))); @@ -1778,8 +1806,8 @@ static RRDDIM *create_rrdim_entry(ONEWAYALLOC *owa, RRDSET *st, char *id, char * rrddim_flag_set(rd, RRDDIM_FLAG_NONE); uuid_copy(rd->metric_uuid, *metric_uuid); - rd->id = onewayalloc_strdupz(owa, id); - rd->name = onewayalloc_strdupz(owa, name); + rd->id = string_strdupz(id); + rd->name = string_strdupz(name); for(int tier = 0; tier < storage_tiers ;tier++) { rd->tiers[tier] = onewayalloc_callocz(owa, 1, sizeof(*rd->tiers[tier])); @@ -1869,8 +1897,9 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) { if (unlikely(st && !st->counter)) { - onewayalloc_freez(owa, st->context); - onewayalloc_freez(owa, (char *) st->name); + string_freez(st->context); + string_freez(st->name); + string_freez(st->id); onewayalloc_freez(owa, st); } st = onewayalloc_callocz(owa, 1, sizeof(*st)); @@ -1879,19 +1908,21 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para snprintfz( n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4), (char *)sqlite3_column_text(res, 3)); - st->name = onewayalloc_strdupz(owa, n); + st->name = string_strdupz(n); st->update_every = sqlite3_column_int(res, 6); st->counter = 0; if (chart) { - st->context = onewayalloc_strdupz(owa, (char *)sqlite3_column_text(res, 8)); - strncpyz(st->id, chart, RRD_ID_LENGTH_MAX); - } + st->context = string_strdupz((char *)sqlite3_column_text(res, 8)); + st->id = string_strdupz(chart); + } else + st->id = string_strdupz(n); + uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7)); st->last_entry_t = 0; st->rrdhost = host; } - if (unlikely(find_dimension_first_last_t(machine_guid, (char *)st->name, (char *)sqlite3_column_text(res, 1), + if (unlikely(find_dimension_first_last_t(machine_guid, (char *)rrdset_name(st), (char *)sqlite3_column_text(res, 1), (uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t, &rrdeng_uuid, 0))) continue; @@ -1909,13 +1940,14 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para } if (st) { if (!st->counter) { - onewayalloc_freez(owa,st->context); - onewayalloc_freez(owa,(char *)st->name); + string_freez(st->context); + string_freez(st->name); + string_freez(st->id); onewayalloc_freez(owa,st); } else if (!st->context && context) - st->context = onewayalloc_strdupz(owa,context); + st->context = string_strdupz(context); } failed: @@ -2063,7 +2095,7 @@ void compute_chart_hash(RRDSET *st) char priority_str[32]; if (rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) { - internal_error(true, "Skipping compute_chart_hash for host %s because context streaming is enabled", st->rrdhost->hostname); + internal_error(true, "Skipping compute_chart_hash for host %s because context streaming is enabled", rrdhost_hostname(st->rrdhost)); return; } @@ -2072,15 +2104,14 @@ void compute_chart_hash(RRDSET *st) evpctx = EVP_MD_CTX_create(); EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL); //EVP_DigestUpdate(evpctx, st->type, strlen(st->type)); - EVP_DigestUpdate(evpctx, st->id, strlen(st->id)); - EVP_DigestUpdate(evpctx, st->name, strlen(st->name)); - EVP_DigestUpdate(evpctx, st->family, strlen(st->family)); - EVP_DigestUpdate(evpctx, st->context, strlen(st->context)); - EVP_DigestUpdate(evpctx, st->title, strlen(st->title)); - EVP_DigestUpdate(evpctx, st->units, strlen(st->units)); - EVP_DigestUpdate(evpctx, st->plugin_name, strlen(st->plugin_name)); - if (st->module_name) - EVP_DigestUpdate(evpctx, st->module_name, strlen(st->module_name)); + EVP_DigestUpdate(evpctx, rrdset_id(st), string_strlen(st->id)); + EVP_DigestUpdate(evpctx, rrdset_name(st), string_strlen(st->name)); + EVP_DigestUpdate(evpctx, rrdset_family(st), string_strlen(st->family)); + EVP_DigestUpdate(evpctx, rrdset_context(st), string_strlen(st->context)); + EVP_DigestUpdate(evpctx, rrdset_title(st), string_strlen(st->title)); + EVP_DigestUpdate(evpctx, rrdset_units(st), string_strlen(st->units)); + EVP_DigestUpdate(evpctx, rrdset_plugin_name(st), string_strlen(st->plugin_name)); + EVP_DigestUpdate(evpctx, rrdset_module_name(st), string_strlen(st->module_name)); // EVP_DigestUpdate(evpctx, priority_str, strlen(priority_str)); EVP_DigestUpdate(evpctx, &st->priority, sizeof(st->priority)); EVP_DigestUpdate(evpctx, &st->chart_type, sizeof(st->chart_type)); @@ -2096,15 +2127,15 @@ void compute_chart_hash(RRDSET *st) (void)sql_store_chart_hash( (uuid_t *)&hash_value, st->chart_uuid, - st->type, - st->id, - st->name, - st->family, - st->context, - st->title, - st->units, - st->plugin_name, - st->module_name, + rrdset_type(st), + rrdset_id(st), + rrdset_name(st), + rrdset_family(st), + rrdset_context(st), + rrdset_title(st), + rrdset_units(st), + rrdset_plugin_name(st), + rrdset_module_name(st), st->priority, st->chart_type); #else @@ -2224,7 +2255,7 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id) char host_guid[GUID_LEN + 1]; uuid_unparse_lower(*host_id, host_guid); rrd_wrlock(); - host = rrdhost_find_by_guid(host_guid, 0); + host = rrdhost_find_by_guid(host_guid); if (likely(host)) set_host_node_id(host, node_id); rrd_unlock(); @@ -2439,7 +2470,7 @@ struct node_instance_list *get_node_list(void) uuid_copy(node_list[row].host_id, *host_id); node_list[row].queryable = 1; uuid_unparse_lower(*host_id, host_guid); - RRDHOST *host = rrdhost_find_by_guid(host_guid, 0); + RRDHOST *host = rrdhost_find_by_guid(host_guid); node_list[row].live = host && (host == localhost || host->receiver) ? 1 : 0; node_list[row].hops = (host && host->system_info) ? host->system_info->hops : uuid_compare(*host_id, localhost->host_uuid) ? 1 : 0; @@ -2677,7 +2708,7 @@ void sql_store_host_labels(RRDHOST *host) { int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid); if (rc != SQLITE_OK) - error_report("Failed to remove old host labels for host %s", host->hostname); + error_report("Failed to remove old host labels for host %s", rrdhost_hostname(host)); rrdlabels_walkthrough_read(host->host_labels, save_host_label_callback, host); } diff --git a/database/sqlite/sqlite_health.c b/database/sqlite/sqlite_health.c index 1eb5c05b02..58ee73b543 100644 --- a/database/sqlite/sqlite_health.c +++ b/database/sqlite/sqlite_health.c @@ -15,7 +15,7 @@ int sql_create_health_log_table(RRDHOST *host) { if (unlikely(!db_meta)) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("HEALTH [%s]: Database has not been initialized", host->hostname); + error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host)); return 1; } @@ -26,7 +26,7 @@ int sql_create_health_log_table(RRDHOST *host) { rc = sqlite3_exec_monitored(db_meta, command, 0, 0, &err_msg); if (rc != SQLITE_OK) { - error_report("HEALTH [%s]: SQLite error during creation of health log table, rc = %d (%s)", host->hostname, rc, err_msg); + error_report("HEALTH [%s]: SQLite error during creation of health log table, rc = %d (%s)", rrdhost_hostname(host), rc, err_msg); sqlite3_free(err_msg); return 1; } @@ -49,7 +49,7 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) { if (unlikely(!db_meta)) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("HEALTH [%s]: Database has not been initialized", host->hostname); + error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host)); return; } @@ -60,7 +60,7 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) { rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0); if (unlikely(rc != SQLITE_OK)) { - error_report("HEALTH [%s]: Failed to prepare statement for SQL_UPDATE_HEALTH_LOG", host->hostname); + error_report("HEALTH [%s]: Failed to prepare statement for SQL_UPDATE_HEALTH_LOG", rrdhost_hostname(host)); return; } @@ -96,12 +96,12 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) { rc = execute_insert(res); if (unlikely(rc != SQLITE_DONE)) { - error_report("HEALTH [%s]: Failed to update health log, rc = %d", host->hostname, rc); + error_report("HEALTH [%s]: Failed to update health log, rc = %d", rrdhost_hostname(host), rc); } failed: if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("HEALTH [%s]: Failed to finalize the prepared statement for updating health log.", host->hostname); + error_report("HEALTH [%s]: Failed to finalize the prepared statement for updating health log.", rrdhost_hostname(host)); return; } @@ -122,7 +122,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) { if (unlikely(!db_meta)) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("HEALTH [%s]: Database has not been initialized", host->hostname); + error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host)); return; } @@ -133,11 +133,11 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) { rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0); if (unlikely(rc != SQLITE_OK)) { - error_report("HEALTH [%s]: Failed to prepare statement for SQL_INSERT_HEALTH_LOG", host->hostname); + error_report("HEALTH [%s]: Failed to prepare statement for SQL_INSERT_HEALTH_LOG", rrdhost_hostname(host)); return; } - rc = sqlite3_bind_text(res, 1, host->hostname, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 1, rrdhost_hostname(host), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind hostname parameter for SQL_INSERT_HEALTH_LOG"); goto failed; @@ -215,49 +215,49 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) { goto failed; } - rc = sqlite3_bind_text(res, 14, ae->name, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 14, ae_name(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind name parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 15, ae->chart, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 15, ae_chart_name(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind chart parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 16, ae->family, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 16, ae_family(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind family parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 17, ae->exec, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 17, ae_exec(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind exec parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 18, ae->recipient, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 18, ae_recipient(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind recipient parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 19, ae->source, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 19, ae_source(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind source parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 20, ae->units, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 20, ae_units(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind host_id parameter to store node instance information"); goto failed; } - rc = sqlite3_bind_text(res, 21, ae->info, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 21, ae_info(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind info parameter for SQL_INSERT_HEALTH_LOG"); goto failed; @@ -305,25 +305,25 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) { goto failed; } - rc = sqlite3_bind_text(res, 29, ae->classification, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 29, ae_classification(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind classification parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 30, ae->component, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 30, ae_component(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind component parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 31, ae->type, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 31, ae_type(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind type parameter for SQL_INSERT_HEALTH_LOG"); goto failed; } - rc = sqlite3_bind_text(res, 32, ae->chart_context, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 32, ae_chart_context(ae), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind chart_context parameter for SQL_INSERT_HEALTH_LOG"); goto failed; @@ -331,7 +331,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) { rc = execute_insert(res); if (unlikely(rc != SQLITE_DONE)) { - error_report("HEALTH [%s]: Failed to execute SQL_INSERT_HEALTH_LOG, rc = %d", host->hostname, rc); + error_report("HEALTH [%s]: Failed to execute SQL_INSERT_HEALTH_LOG, rc = %d", rrdhost_hostname(host), rc); goto failed; } @@ -340,7 +340,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) { failed: if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("HEALTH [%s]: Failed to finalize the prepared statement for inserting to health log.", host->hostname); + error_report("HEALTH [%s]: Failed to finalize the prepared statement for inserting to health log.", rrdhost_hostname(host)); return; } @@ -436,7 +436,7 @@ void sql_health_alarm_log_count(RRDHOST *host) { if (unlikely(rc != SQLITE_OK)) error_report("Failed to finalize the prepared statement to count health log entries from db"); - info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", host->hostname, uuid_str, host->health_log_entries_written); + info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", rrdhost_hostname(host), uuid_str, host->health_log_entries_written); } #define SQL_INJECT_REMOVED(guid, guid2) "insert into health_log_%s (hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, " \ @@ -607,7 +607,7 @@ void sql_check_removed_alerts_state(char *uuid_str) #define SQL_LOAD_HEALTH_LOG(guid,limit) "SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type, chart_context FROM (SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type, chart_context FROM health_log_%s order by unique_id desc limit %u) order by unique_id asc;", guid, limit void sql_health_alarm_log_load(RRDHOST *host) { sqlite3_stmt *res = NULL; - int rc; + int ret; ssize_t errored = 0, loaded = 0; char command[MAX_HEALTH_SQL_SIZE + 1]; @@ -615,7 +615,7 @@ void sql_health_alarm_log_load(RRDHOST *host) { if (unlikely(!db_meta)) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("HEALTH [%s]: Database has not been initialized", host->hostname); + error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host)); return; } @@ -626,47 +626,52 @@ void sql_health_alarm_log_load(RRDHOST *host) { snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_LOAD_HEALTH_LOG(uuid_str, host->health_log.max)); - rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("HEALTH [%s]: Failed to prepare sql statement to load health log.", host->hostname); + ret = sqlite3_prepare_v2(db_meta, command, -1, &res, 0); + if (unlikely(ret != SQLITE_OK)) { + error_report("HEALTH [%s]: Failed to prepare sql statement to load health log.", rrdhost_hostname(host)); return; } netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); + DICTIONARY *all_rrdcalcs = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + RRDCALC *rc; + foreach_rrdcalc_in_rrdhost(host, rc) + dictionary_set(all_rrdcalcs, rrdcalc_name(rc), rc, sizeof(*rc)); + while (sqlite3_step_monitored(res) == SQLITE_ROW) { ALARM_ENTRY *ae = NULL; // check that we have valid ids uint32_t unique_id = (uint32_t) sqlite3_column_int64(res, 1); if(!unique_id) { - error_report("HEALTH [%s]: Got invalid unique id. Ignoring it.", host->hostname); + error_report("HEALTH [%s]: Got invalid unique id. Ignoring it.", rrdhost_hostname(host)); errored++; continue; } uint32_t alarm_id = (uint32_t) sqlite3_column_int64(res, 2); if(!alarm_id) { - error_report("HEALTH [%s]: Got invalid alarm id. Ignoring it.", host->hostname); + error_report("HEALTH [%s]: Got invalid alarm id. Ignoring it.", rrdhost_hostname(host)); errored++; continue; } //need name, chart and family if (sqlite3_column_type(res, 13) == SQLITE_NULL) { - error_report("HEALTH [%s]: Got null name field. Ignoring it.", host->hostname); + error_report("HEALTH [%s]: Got null name field. Ignoring it.", rrdhost_hostname(host)); errored++; continue; } if (sqlite3_column_type(res, 14) == SQLITE_NULL) { - error_report("HEALTH [%s]: Got null chart field. Ignoring it.", host->hostname); + error_report("HEALTH [%s]: Got null chart field. Ignoring it.", rrdhost_hostname(host)); errored++; continue; } if (sqlite3_column_type(res, 15) == SQLITE_NULL) { - error_report("HEALTH [%s]: Got null family field. Ignoring it.", host->hostname); + error_report("HEALTH [%s]: Got null family field. Ignoring it.", rrdhost_hostname(host)); errored++; continue; } @@ -675,18 +680,7 @@ void sql_health_alarm_log_load(RRDHOST *host) { time_t last_repeat = 0; last_repeat = (time_t)sqlite3_column_int64(res, 27); - RRDCALC *rc = alarm_max_last_repeat(host, (char *) sqlite3_column_text(res, 14), simple_hash((char *) sqlite3_column_text(res, 14))); - if (!rc) { - for(rc = host->alarms; rc ; rc = rc->next) { - RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_name, (avl_t *)rc); - if(rdcmp != rc) { - error("Cannot insert the alarm index ID using log %s", rc->name); - } - } - - rc = alarm_max_last_repeat(host, (char *) sqlite3_column_text(res, 14), simple_hash((char *) sqlite3_column_text(res, 14))); - } - + rc = dictionary_get(all_rrdcalcs, (char *) sqlite3_column_text(res, 14)); if(unlikely(rc)) { if (rrdcalc_isrepeating(rc)) { rc->last_repeat = last_repeat; @@ -719,36 +713,32 @@ void sql_health_alarm_log_load(RRDHOST *host) { ae->exec_run_timestamp = (time_t) sqlite3_column_int64(res, 11); ae->delay_up_to_timestamp = (time_t) sqlite3_column_int64(res, 12); - ae->name = strdupz((char *) sqlite3_column_text(res, 13)); - ae->hash_name = simple_hash(ae->name); - - ae->chart = strdupz((char *) sqlite3_column_text(res, 14)); - ae->hash_chart = simple_hash(ae->chart); - - ae->family = strdupz((char *) sqlite3_column_text(res, 15)); + ae->name = string_strdupz((char *) sqlite3_column_text(res, 13)); + ae->chart = string_strdupz((char *) sqlite3_column_text(res, 14)); + ae->family = string_strdupz((char *) sqlite3_column_text(res, 15)); if (sqlite3_column_type(res, 16) != SQLITE_NULL) - ae->exec = strdupz((char *) sqlite3_column_text(res, 16)); + ae->exec = string_strdupz((char *) sqlite3_column_text(res, 16)); else ae->exec = NULL; if (sqlite3_column_type(res, 17) != SQLITE_NULL) - ae->recipient = strdupz((char *) sqlite3_column_text(res, 17)); + ae->recipient = string_strdupz((char *) sqlite3_column_text(res, 17)); else ae->recipient = NULL; if (sqlite3_column_type(res, 18) != SQLITE_NULL) - ae->source = strdupz((char *) sqlite3_column_text(res, 18)); + ae->source = string_strdupz((char *) sqlite3_column_text(res, 18)); else ae->source = NULL; if (sqlite3_column_type(res, 19) != SQLITE_NULL) - ae->units = strdupz((char *) sqlite3_column_text(res, 19)); + ae->units = string_strdupz((char *) sqlite3_column_text(res, 19)); else ae->units = NULL; if (sqlite3_column_type(res, 20) != SQLITE_NULL) - ae->info = strdupz((char *) sqlite3_column_text(res, 20)); + ae->info = string_strdupz((char *) sqlite3_column_text(res, 20)); else ae->info = NULL; @@ -763,30 +753,30 @@ void sql_health_alarm_log_load(RRDHOST *host) { ae->last_repeat = last_repeat; if (sqlite3_column_type(res, 28) != SQLITE_NULL) - ae->classification = strdupz((char *) sqlite3_column_text(res, 28)); + ae->classification = string_strdupz((char *) sqlite3_column_text(res, 28)); else ae->classification = NULL; if (sqlite3_column_type(res, 29) != SQLITE_NULL) - ae->component = strdupz((char *) sqlite3_column_text(res, 29)); + ae->component = string_strdupz((char *) sqlite3_column_text(res, 29)); else ae->component = NULL; if (sqlite3_column_type(res, 30) != SQLITE_NULL) - ae->type = strdupz((char *) sqlite3_column_text(res, 30)); + ae->type = string_strdupz((char *) sqlite3_column_text(res, 30)); else ae->type = NULL; if (sqlite3_column_type(res, 31) != SQLITE_NULL) - ae->chart_context = strdupz((char *) sqlite3_column_text(res, 31)); + ae->chart_context = string_strdupz((char *) sqlite3_column_text(res, 31)); else ae->chart_context = NULL; char value_string[100 + 1]; - freez(ae->old_value_string); - freez(ae->new_value_string); - ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1)); - ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1)); + string_freez(ae->old_value_string); + string_freez(ae->new_value_string); + ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1)); + ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1)); ae->next = host->health_log.alarms; host->health_log.alarms = ae; @@ -800,6 +790,9 @@ void sql_health_alarm_log_load(RRDHOST *host) { loaded++; } + dictionary_destroy(all_rrdcalcs); + all_rrdcalcs = NULL; + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); if(!host->health_max_unique_id) host->health_max_unique_id = (uint32_t)now_realtime_sec(); @@ -809,10 +802,10 @@ void sql_health_alarm_log_load(RRDHOST *host) { if (unlikely(!host->health_log.next_alarm_id || host->health_log.next_alarm_id <= host->health_max_alarm_id)) host->health_log.next_alarm_id = host->health_max_alarm_id + 1; - info("HEALTH [%s]: Table health_log_%s, loaded %zd alarm entries, errors in %zd entries.", host->hostname, uuid_str, loaded, errored); + info("HEALTH [%s]: Table health_log_%s, loaded %zd alarm entries, errors in %zd entries.", rrdhost_hostname(host), uuid_str, loaded, errored); - rc = sqlite3_finalize(res); - if (unlikely(rc != SQLITE_OK)) + ret = sqlite3_finalize(res); + if (unlikely(ret != SQLITE_OK)) error_report("Failed to finalize the health log read statement"); sql_health_alarm_log_count(host); @@ -854,154 +847,154 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg) goto bind_fail; param++; - if (cfg->alarm && *cfg->alarm) - rc = sqlite3_bind_text(res, 2, cfg->alarm, -1, SQLITE_STATIC); + if (cfg->alarm) + rc = sqlite3_bind_text(res, 2, string2str(cfg->alarm), -1, SQLITE_STATIC); else rc = sqlite3_bind_null(res, 2); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - if (cfg->template_key && *cfg->template_key) - rc = sqlite3_bind_text(res, 3, cfg->template_key, -1, SQLITE_STATIC); + if (cfg->template_key) + rc = sqlite3_bind_text(res, 3, string2str(cfg->template_key), -1, SQLITE_STATIC); else rc = sqlite3_bind_null(res, 3); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 4, cfg->on, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 4, string2str(cfg->on), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 5, cfg->classification, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 5, string2str(cfg->classification), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 6, cfg->component, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 6, string2str(cfg->component), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 7, cfg->type, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 7, string2str(cfg->type), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 8, cfg->os, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 8, string2str(cfg->os), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 9, cfg->host, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 9, string2str(cfg->host), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 10, cfg->lookup, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 10, string2str(cfg->lookup), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 11, cfg->every, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 11, string2str(cfg->every), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 12, cfg->units, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 12, string2str(cfg->units), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 13, cfg->calc, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 13, string2str(cfg->calc), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 14, cfg->families, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 14, string2str(cfg->families), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 15, cfg->plugin, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 15, string2str(cfg->plugin), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 16, cfg->module, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 16, string2str(cfg->module), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 17, cfg->charts, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 17, string2str(cfg->charts), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 18, cfg->green, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 18, string2str(cfg->green), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 19, cfg->red, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 19, string2str(cfg->red), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 20, cfg->warn, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 20, string2str(cfg->warn), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 21, cfg->crit, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 21, string2str(cfg->crit), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 22, cfg->exec, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 22, string2str(cfg->exec), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 23, cfg->to, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 23, string2str(cfg->to), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 24, cfg->info, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 24, string2str(cfg->info), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 25, cfg->delay, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 25, string2str(cfg->delay), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 26, cfg->options, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 26, string2str(cfg->options), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 27, cfg->repeat, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 27, string2str(cfg->repeat), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 28, cfg->host_labels, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 28, string2str(cfg->host_labels), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; if (cfg->p_db_lookup_after) { param++; - rc = sqlite3_bind_text(res, 29, cfg->p_db_lookup_dimensions, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 29, string2str(cfg->p_db_lookup_dimensions), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; param++; - rc = sqlite3_bind_text(res, 30, cfg->p_db_lookup_method, -1, SQLITE_STATIC); + rc = sqlite3_bind_text(res, 30, string2str(cfg->p_db_lookup_method), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; @@ -1071,7 +1064,7 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg) skip hash calculations */ #if !defined DISABLE_CLOUD && defined ENABLE_HTTPS -#define DIGEST_ALERT_CONFIG_VAL(v) ((v) ? EVP_DigestUpdate(evpctx, (v), strlen((v))) : EVP_DigestUpdate(evpctx, "", 1)) +#define DIGEST_ALERT_CONFIG_VAL(v) EVP_DigestUpdate(evpctx, (string2str(v)), string_strlen(v)) #endif int alert_hash_and_store_config( uuid_t hash_id, diff --git a/exporting/check_filters.c b/exporting/check_filters.c index 726fd02a1a..b07ec07b17 100644 --- a/exporting/check_filters.c +++ b/exporting/check_filters.c @@ -25,7 +25,7 @@ int rrdhost_is_exportable(struct instance *instance, RRDHOST *host) RRDHOST_FLAGS *flags = &host->exporting_flags[instance->index]; if (unlikely((*flags & (RRDHOST_FLAG_EXPORTING_SEND | RRDHOST_FLAG_EXPORTING_DONT_SEND)) == 0)) { - char *host_name = (host == localhost) ? "localhost" : host->hostname; + const char *host_name = (host == localhost) ? "localhost" : rrdhost_hostname(host); if (!instance->config.hosts_pattern || simple_pattern_matches(instance->config.hosts_pattern, host_name)) { *flags |= RRDHOST_FLAG_EXPORTING_SEND; @@ -69,22 +69,22 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st) if(unlikely(!(*flags & RRDSET_FLAG_EXPORTING_SEND))) { // we have not checked this chart - if(simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name)) + if(simple_pattern_matches(instance->config.charts_pattern, rrdset_id(st)) || simple_pattern_matches(instance->config.charts_pattern, rrdset_name(st))) *flags |= RRDSET_FLAG_EXPORTING_SEND; else { *flags |= RRDSET_FLAG_EXPORTING_IGNORE; - debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", st->id, host->hostname); + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", rrdset_id(st), rrdhost_hostname(host)); return 0; } } if(unlikely(!rrdset_is_available_for_exporting_and_alarms(st))) { - debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", st->id, host->hostname); + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", rrdset_id(st), rrdhost_hostname(host)); return 0; } if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) { - debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode)); + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode)); return 0; } diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c index 8ca094b3b7..6375ec2094 100644 --- a/exporting/graphite/graphite.c +++ b/exporting/graphite/graphite.c @@ -123,24 +123,24 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); buffer_sprintf( instance->buffer, "%s.%s.%s.%s%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n", instance->config.prefix, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), chart_name, dimension_name, (host->tags) ? ";" : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "", rd->last_collected_value, (unsigned long long)rd->last_collected_time.tv_sec); @@ -163,13 +163,13 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); time_t last_t; @@ -182,11 +182,11 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM instance->buffer, "%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n", instance->config.prefix, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), chart_name, dimension_name, (host->tags) ? ";" : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "", value, (unsigned long long)last_t); diff --git a/exporting/json/json.c b/exporting/json/json.c index 45a8c9d9fe..627b8208e7 100644 --- a/exporting/json/json.c +++ b/exporting/json/json.c @@ -145,7 +145,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM RRDSET *st = rd->rrdset; RRDHOST *host = st->rrdhost; - const char *tags_pre = "", *tags_post = "", *tags = host->tags; + const char *tags_pre = "", *tags_post = "", *tags = rrdhost_tags(host); if (!tags) tags = ""; @@ -187,21 +187,20 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM "\"timestamp\":%llu}", instance->config.prefix, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), tags_pre, tags, tags_post, instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "", - st->id, - st->name, - st->family, - st->context, - st->type, - st->units, - - rd->id, - rd->name, + rrdset_id(st), + rrdset_name(st), + rrdset_family(st), + rrdset_context(st), + rrdset_type(st), + rrdset_units(st), + rrddim_id(rd), + rrddim_name(rd), rd->last_collected_value, (unsigned long long)rd->last_collected_time.tv_sec); @@ -231,7 +230,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd if(isnan(value)) return 0; - const char *tags_pre = "", *tags_post = "", *tags = host->tags; + const char *tags_pre = "", *tags_post = "", *tags = rrdhost_tags(host); if (!tags) tags = ""; @@ -272,21 +271,20 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd "\"timestamp\": %llu}", instance->config.prefix, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), tags_pre, tags, tags_post, instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "", - st->id, - st->name, - st->family, - st->context, - st->type, - st->units, - - rd->id, - rd->name, + rrdset_id(st), + rrdset_name(st), + rrdset_family(st), + rrdset_context(st), + rrdset_type(st), + rrdset_units(st), + rrddim_id(rd), + rrddim_name(rd), value, (unsigned long long)last_t); diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c index 282de2e6b9..ed56e3d253 100644 --- a/exporting/opentsdb/opentsdb.c +++ b/exporting/opentsdb/opentsdb.c @@ -177,13 +177,13 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); buffer_sprintf( @@ -194,9 +194,9 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM dimension_name, (unsigned long long)rd->last_collected_time.tv_sec, rd->last_collected_value, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (host->tags) ? " " : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : ""); return 0; @@ -217,13 +217,13 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); time_t last_t; @@ -240,9 +240,9 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r dimension_name, (unsigned long long)last_t, value, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (host->tags) ? " " : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : ""); return 0; @@ -309,13 +309,13 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM * char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); if (buffer_strlen((BUFFER *)instance->buffer) > 2) @@ -336,9 +336,9 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM * dimension_name, (unsigned long long)rd->last_collected_time.tv_sec, rd->last_collected_value, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (host->tags) ? " " : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : ""); return 0; @@ -359,13 +359,13 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd) char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); time_t last_t; @@ -392,9 +392,9 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd) dimension_name, (unsigned long long)last_t, value, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (host->tags) ? " " : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : ""); return 0; diff --git a/exporting/process_data.c b/exporting/process_data.c index d5138b787b..e1e939c51d 100644 --- a/exporting/process_data.c +++ b/exporting/process_data.c @@ -110,9 +110,9 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data( debug( D_EXPORTING, "EXPORTING: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu", - host->hostname, - st->id, - rd->id, + rrdhost_hostname(host), + rrdset_id(st), + rrddim_id(rd), (unsigned long)after, (unsigned long)before, (unsigned long)first_t, @@ -142,9 +142,9 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data( debug( D_EXPORTING, "EXPORTING: %s.%s.%s: no values stored in database for range %lu to %lu", - host->hostname, - st->id, - rd->id, + rrdhost_hostname(host), + rrdset_id(st), + rrddim_id(rd), (unsigned long)after, (unsigned long)before); return NAN; diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c index 7d632164f6..560022b72f 100644 --- a/exporting/prometheus/prometheus.c +++ b/exporting/prometheus/prometheus.c @@ -9,9 +9,9 @@ static int is_matches_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN *filter) { if (instance->config.options & EXPORTING_OPTION_SEND_NAMES) { - return simple_pattern_matches(filter, st->name); + return simple_pattern_matches(filter, rrdset_name(st)); } - return simple_pattern_matches(filter, st->id); + return simple_pattern_matches(filter, rrdset_id(st)); } /** @@ -48,8 +48,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN debug( D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", - st->id, - host->hostname); + rrdset_id(st), + rrdhost_hostname(host)); return 0; } } @@ -58,8 +58,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN debug( D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", - st->id, - host->hostname); + rrdset_id(st), + rrdhost_hostname(host)); return 0; } @@ -69,8 +69,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN debug( D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting connector requires database access.", - st->id, - host->hostname, + rrdset_id(st), + rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode)); return 0; } @@ -349,8 +349,9 @@ struct host_variables_callback_options { * @param data callback options. * @return Returns 1 if the chart can be sent, 0 otherwise. */ -static int print_host_variables(RRDVAR *rv, void *data) -{ +static int print_host_variables(const char *name __maybe_unused, void *rv_ptr, void *data) { + RRDVAR *rv = rv_ptr; + struct host_variables_callback_options *opts = data; if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) { @@ -366,7 +367,7 @@ static int print_host_variables(RRDVAR *rv, void *data) if (isnan(value) || isinf(value)) { if (opts->output_options & PROMETHEUS_OUTPUT_HELP) buffer_sprintf( - opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value)) ? "NAN" : "INF"); + opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rrdvar_name(rv), (isnan(value)) ? "NAN" : "INF"); return 0; } @@ -378,7 +379,7 @@ static int print_host_variables(RRDVAR *rv, void *data) label_post = "}"; } - prometheus_name_copy(opts->name, rv->name, sizeof(opts->name)); + prometheus_name_copy(opts->name, rrdvar_name(rv), sizeof(opts->name)); if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) buffer_sprintf( @@ -445,17 +446,17 @@ static void generate_as_collected_prom_help(BUFFER *wb, struct gen_parameters *p wb, "%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * ", p->suffix, - (p->output_options & PROMETHEUS_OUTPUT_NAMES && p->st->name) ? p->st->name : p->st->id, - p->st->context, - p->st->family, - (p->output_options & PROMETHEUS_OUTPUT_NAMES && p->rd->name) ? p->rd->name : p->rd->id); + (p->output_options & PROMETHEUS_OUTPUT_NAMES && p->st->name) ? rrdset_name(p->st) : rrdset_id(p->st), + rrdset_context(p->st), + rrdset_family(p->st), + (p->output_options & PROMETHEUS_OUTPUT_NAMES && p->rd->name) ? rrddim_name(p->rd) : rrddim_id(p->rd)); if (prometheus_collector) buffer_sprintf(wb, "1 / 1"); else buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT, p->rd->multiplier, p->rd->divisor); - buffer_sprintf(wb, " %s %s (%s)\n", p->relation, p->st->units, p->type); + buffer_sprintf(wb, " %s %s (%s)\n", p->relation, rrdset_units(p->st), p->type); } /** @@ -521,7 +522,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( rrdhost_rdlock(host); char hostname[PROMETHEUS_ELEMENT_MAX + 1]; - prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(hostname, rrdhost_hostname(host), PROMETHEUS_ELEMENT_MAX); format_host_labels_prometheus(instance, host); @@ -529,8 +530,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"", hostname, - host->program_name, - host->program_version); + rrdhost_program_name(host), + rrdhost_program_version(host)); if (instance->labels_buffer && *buffer_tostring(instance->labels_buffer)) { buffer_sprintf(wb, ",%s", buffer_tostring(instance->labels_buffer)); @@ -551,15 +552,19 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( // send custom variables set for the host if (output_options & PROMETHEUS_OUTPUT_VARIABLES) { - struct host_variables_callback_options opts = { .host = host, - .wb = wb, - .labels = (labels[0] == ',') ? &labels[1] : labels, - .exporting_options = exporting_options, - .output_options = output_options, - .prefix = prefix, - .now = now_realtime_sec(), - .host_header_printed = 0 }; - foreach_host_variable_callback(host, print_host_variables, &opts); + + struct host_variables_callback_options opts = { + .host = host, + .wb = wb, + .labels = (labels[0] == ',') ? &labels[1] : labels, + .exporting_options = exporting_options, + .output_options = output_options, + .prefix = prefix, + .now = now_realtime_sec(), + .host_header_printed = 0 + }; + + rrdvar_walkthrough_read(host->rrdvar_root_index, print_host_variables, &opts); } // for each chart @@ -575,10 +580,9 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( char family[PROMETHEUS_ELEMENT_MAX + 1]; char units[PROMETHEUS_ELEMENT_MAX + 1] = ""; - prometheus_label_copy( - chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, PROMETHEUS_ELEMENT_MAX); - prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); - prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, rrdset_family(st), PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, rrdset_context(st), PROMETHEUS_ELEMENT_MAX); int as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AS_COLLECTED); int homogeneous = 1; @@ -590,13 +594,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( if (rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) homogeneous = 0; - if (st->module_name && !strcmp(st->module_name, "prometheus")) + if (!strcmp(rrdset_module_name(st), "prometheus")) prometheus_collector = 1; } else { if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE && !(output_options & PROMETHEUS_OUTPUT_HIDEUNITS)) prometheus_units_copy( - units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS); + units, rrdset_units(st), PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS); } if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) @@ -604,10 +608,10 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n", (homogeneous) ? "homogeneous" : "heterogeneous", - (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, - st->context, - st->family, - st->units); + (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), + rrdset_context(st), + rrdset_family(st), + rrdset_units(st)); // for each dimension RRDDIM *rd; @@ -651,7 +655,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( prometheus_label_copy( dimension, - (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, + (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) @@ -667,7 +671,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( prometheus_name_copy( dimension, - (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, + (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) @@ -694,7 +698,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( prometheus_label_copy( dimension, - (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, + (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) @@ -705,8 +709,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( context, units, suffix, - (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, - st->units, + (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), + rrdset_units(st), (unsigned long long)first_time, (unsigned long long)last_time); @@ -809,7 +813,7 @@ static inline time_t prometheus_preparation( buffer_sprintf( wb, "# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s, time range %lu to %lu\n\n", - host->hostname, + rrdhost_hostname(host), (first_seen) ? "FIRST SEEN " : "", server, mode, diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c index 03feb2c08b..29215283e8 100644 --- a/exporting/prometheus/remote_write/remote_write.c +++ b/exporting/prometheus/remote_write/remote_write.c @@ -171,12 +171,12 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host char hostname[PROMETHEUS_ELEMENT_MAX + 1]; prometheus_label_copy( hostname, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), PROMETHEUS_ELEMENT_MAX); add_host_info( connector_specific_data->write_request, - "netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS); + "netdata_info", hostname, rrdhost_program_name(host), rrdhost_program_version(host), now_realtime_usec() / USEC_PER_MS); if (unlikely(sending_labels_configured(instance))) { struct format_remote_write_label_callback tmp = { @@ -200,10 +200,10 @@ int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st) { prometheus_label_copy( chart, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), PROMETHEUS_ELEMENT_MAX); - prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); - prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, rrdset_family(st), PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, rrdset_context(st), PROMETHEUS_ELEMENT_MAX); as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED); homogeneous = 1; @@ -215,7 +215,7 @@ int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st) homogeneous = 0; } else { if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE) - prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0); + prometheus_units_copy(units, rrdset_units(st), PROMETHEUS_ELEMENT_MAX, 0); } return 0; @@ -249,8 +249,8 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * D_EXPORTING, "EXPORTING: not sending dimension '%s' of chart '%s' from host '%s', " "its last data collection (%lu) is not within our timeframe (%lu to %lu)", - rd->id, rd->rrdset->id, - (host == localhost) ? instance->config.hostname : host->hostname, + rrddim_id(rd), rrdset_id(rd->rrdset), + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)instance->after, (unsigned long)instance->before); @@ -263,14 +263,14 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * prometheus_label_copy( dimension, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix); add_metric( connector_specific_data->write_request, name, chart, family, dimension, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), rd->last_collected_value, timeval_msec(&rd->last_collected_time)); } else { // the dimensions of the chart, do not have the same algorithm, multiplier or divisor @@ -278,7 +278,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * prometheus_name_copy( dimension, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); snprintf( name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", instance->config.prefix, context, dimension, @@ -287,7 +287,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * add_metric( connector_specific_data->write_request, name, chart, family, NULL, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), rd->last_collected_value, timeval_msec(&rd->last_collected_time)); } } else { @@ -304,7 +304,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * prometheus_label_copy( dimension, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); snprintf( name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", instance->config.prefix, context, units, suffix); @@ -312,7 +312,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * add_metric( connector_specific_data->write_request, name, chart, family, dimension, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), value, last_t * MSEC_PER_SEC); } } @@ -321,7 +321,9 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * return 0; } -int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) { +static int format_variable_prometheus_remote_write_callback(const char *name_txt __maybe_unused, void *rv_ptr, void *data) { + RRDVAR *rv = rv_ptr; + struct prometheus_remote_write_variables_callback_options *opts = data; if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) { @@ -335,12 +337,12 @@ int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) { char name[PROMETHEUS_LABELS_MAX + 1]; char *suffix = ""; - prometheus_name_copy(context, rv->name, PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, rrdvar_name(rv), PROMETHEUS_ELEMENT_MAX); snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix); NETDATA_DOUBLE value = rrdvar2number(rv); add_variable(connector_specific_data->write_request, name, - (host == localhost) ? instance->config.hostname : host->hostname, value, opts->now / USEC_PER_MS); + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), value, opts->now / USEC_PER_MS); } return 0; @@ -361,7 +363,7 @@ int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST .now = now_realtime_usec(), }; - return foreach_host_variable_callback(host, format_variable_prometheus_remote_write_callback, &opt); + return rrdvar_walkthrough_read(host->rrdvar_root_index, format_variable_prometheus_remote_write_callback, &opt); } /** diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c index aae1c53fb2..da48e424be 100644 --- a/exporting/tests/exporting_fixtures.c +++ b/exporting/tests/exporting_fixtures.c @@ -39,7 +39,7 @@ int setup_rrdhost() localhost->rrd_update_every = 1; - localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2"); + localhost->tags = string_strdupz("TAG1=VALUE1 TAG2=VALUE2"); localhost->host_labels = rrdlabels_create(); rrdlabels_add(localhost->host_labels, "key1", "value1", RRDLABEL_SRC_CONFIG); @@ -48,16 +48,16 @@ int setup_rrdhost() localhost->rrdset_root = calloc(1, sizeof(RRDSET)); RRDSET *st = localhost->rrdset_root; st->rrdhost = localhost; - strcpy(st->id, "chart_id"); - st->name = strdupz("chart_name"); + st->id = string_strdupz("chart_id"); + st->name = string_strdupz("chart_name"); st->rrd_memory_mode |= RRD_MEMORY_MODE_SAVE; st->update_every = 1; localhost->rrdset_root->dimensions = calloc(1, sizeof(RRDDIM)); RRDDIM *rd = localhost->rrdset_root->dimensions; rd->rrdset = st; - rd->id = strdupz("dimension_id"); - rd->name = strdupz("dimension_name"); + rd->id = string_strdupz("dimension_id"); + rd->name = string_strdupz("dimension_name"); rd->last_collected_value = 123000321; rd->last_collected_time.tv_sec = 15051; rd->collections_counter++; @@ -77,18 +77,18 @@ int setup_rrdhost() int teardown_rrdhost() { RRDDIM *rd = localhost->rrdset_root->dimensions; - free((void *)rd->name); - free((void *)rd->id); + string_freez(rd->name); + string_freez(rd->id); free(rd->tiers[0]); free(rd); RRDSET *st = localhost->rrdset_root; - free((void *)st->name); + string_freez(st->name); free(st); rrdlabels_destroy(localhost->host_labels); - free((void *)localhost->tags); + string_freez(localhost->tags); free(localhost); return 0; diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c index 86707e16e3..735925d478 100644 --- a/exporting/tests/netdata_doubles.c +++ b/exporting/tests/netdata_doubles.c @@ -177,12 +177,6 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) return RRD_MEMORY_MODE_NONE_NAME; } -NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) -{ - (void)rv; - return 0; -} - int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data) { (void)host; @@ -256,6 +250,18 @@ void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, cha (void)value; } -void rrdcalc_update_rrdlabels(RRDSET *st) { +void rrdcalc_update_rrdlabels(RRDSET *st) +{ (void)st; } + +void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) +{ + (void)host; + (void)rv; +} + +void db_execute(const char *cmd) +{ + (void)cmd; +} diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c index 56a28059fb..2eee049a81 100644 --- a/exporting/tests/test_exporting_engine.c +++ b/exporting/tests/test_exporting_engine.c @@ -442,8 +442,8 @@ static void test_format_dimension_collected_json_plaintext(void **state) assert_string_equal( buffer_tostring(engine->instance_root->buffer), "{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\"," - "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\"," - "\"chart_context\":\"(null)\",\"chart_type\":\"(null)\",\"units\":\"(null)\",\"id\":\"dimension_id\"," + "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"\"," + "\"chart_context\":\"\",\"chart_type\":\"\",\"units\":\"\",\"id\":\"dimension_id\"," "\"name\":\"dimension_name\",\"value\":123000321,\"timestamp\":15051}\n"); } @@ -459,8 +459,8 @@ static void test_format_dimension_stored_json_plaintext(void **state) assert_string_equal( buffer_tostring(engine->instance_root->buffer), "{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\"," - "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\"," \ - "\"chart_context\": \"(null)\",\"chart_type\":\"(null)\",\"units\": \"(null)\",\"id\":\"dimension_id\"," + "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"\"," \ + "\"chart_context\": \"\",\"chart_type\":\"\",\"units\": \"\",\"id\":\"dimension_id\"," "\"name\":\"dimension_name\",\"value\":690565856.0000000,\"timestamp\": 15052}\n"); } @@ -1055,9 +1055,9 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) BUFFER *buffer = buffer_create(0); - localhost->hostname = strdupz("test_hostname"); - localhost->rrdset_root->family = strdupz("test_family"); - localhost->rrdset_root->context = strdupz("test_context"); + localhost->hostname = string_strdupz("test_hostname"); + localhost->rrdset_root->family = string_strdupz("test_family"); + localhost->rrdset_root->context = string_strdupz("test_context"); expect_function_call(__wrap_now_realtime_sec); will_return(__wrap_now_realtime_sec, 2); @@ -1069,7 +1069,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n" "test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\"} 690565856.0000000\n"); buffer_flush(buffer); @@ -1085,7 +1085,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n" "# TYPE test_prefix_test_context gauge\n" "test_prefix_test_context{chart=\"chart_name\",family=\"test_family\",dimension=\"dimension_name\"} 690565856.0000000\n"); @@ -1101,7 +1101,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n" "test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\",instance=\"test_hostname\"} 690565856.0000000\n"); free(localhost->rrdset_root->context); @@ -1207,8 +1207,8 @@ static void test_format_host_prometheus_remote_write(void **state) simple_connector_data->connector_specific_data = (void *)connector_specific_data; connector_specific_data->write_request = (void *)0xff; - localhost->program_name = strdupz("test_program"); - localhost->program_version = strdupz("test_version"); + localhost->program_name = string_strdupz("test_program"); + localhost->program_version = string_strdupz("test_version"); expect_function_call(__wrap_add_host_info); expect_value(__wrap_add_host_info, write_request_p, 0xff); diff --git a/exporting/tests/test_exporting_engine.h b/exporting/tests/test_exporting_engine.h index ae0b7df9ae..5443c6018d 100644 --- a/exporting/tests/test_exporting_engine.h +++ b/exporting/tests/test_exporting_engine.h @@ -4,6 +4,7 @@ #define TEST_EXPORTING_ENGINE_H 1 #include "libnetdata/libnetdata.h" +#include "database/rrdvar.h" #include "exporting/exporting_engine.h" #include "exporting/graphite/graphite.h" diff --git a/health/health.c b/health/health.c index 9eb36a9c60..c1045c3d81 100644 --- a/health/health.c +++ b/health/health.c @@ -155,18 +155,11 @@ static void health_reload_host(RRDHOST *host) { // free all running alarms rrdhost_wrlock(host); - while(host->templates) - rrdcalctemplate_unlink_and_free(host, host->templates); + while(host->alarms_templates) + rrdcalctemplate_unlink_and_free(host, host->alarms_templates); - RRDCALCTEMPLATE *rt,*next; - for(rt = host->alarms_template_with_foreach; rt ; rt = next) { - next = rt->next; - rrdcalctemplate_free(rt); - } - host->alarms_template_with_foreach = NULL; - - while(host->alarms) - rrdcalc_unlink_and_free(host, host->alarms); + while(host->host_alarms) + rrdcalc_unlink_and_free(host, host->host_alarms); RRDCALC *rc,*nc; for(rc = host->alarms_with_foreach; rc ; rc = nc) { @@ -266,13 +259,13 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) { // do not send notifications for internal statuses - debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status)); + debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); goto done; } if(unlikely(ae->new_status <= RRDCALC_STATUS_CLEAR && (ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) { // do not send notifications for disabled statuses - debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status)); + debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)", ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); // mark it as run, so that we will send the same alarm if it happens again goto done; } @@ -292,7 +285,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { // we have executed this alarm notification in the past if(t && t->new_status == ae->new_status) { // don't send the notification for the same status again - debug(D_HEALTH, "Health not sending again notification for alarm '%s.%s' status %s", ae->chart, ae->name + debug(D_HEALTH, "Health not sending again notification for alarm '%s.%s' status %s", ae_chart_name(ae), ae_name(ae) , rrdcalc_status2string(ae->new_status)); goto done; } @@ -303,7 +296,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { if(unlikely(ae->new_status == RRDCALC_STATUS_CLEAR)) { if((!(ae->flags & HEALTH_ENTRY_RUN_ONCE)) || (ae->flags & HEALTH_ENTRY_RUN_ONCE && ae->old_status < RRDCALC_STATUS_RAISED) ) { debug(D_HEALTH, "Health not sending notification for first initialization of alarm '%s.%s' status %s" - , ae->chart, ae->name, rrdcalc_status2string(ae->new_status)); + , ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); goto done; } } @@ -312,14 +305,14 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { // Check if alarm notifications are silenced if (ae->flags & HEALTH_ENTRY_FLAG_SILENCED) { - info("Health not sending notification for alarm '%s.%s' status %s (command API has disabled notifications)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status)); + info("Health not sending notification for alarm '%s.%s' status %s (command API has disabled notifications)", ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); goto done; } static char command_to_run[ALARM_EXEC_COMMAND_LENGTH + 1]; - const char *exec = (ae->exec) ? ae->exec : host->health_default_exec; - const char *recipient = (ae->recipient) ? ae->recipient : host->health_default_recipient; + const char *exec = (ae->exec) ? ae_exec(ae) : string2str(host->health_default_exec); + const char *recipient = (ae->recipient) ? ae_recipient(ae) : string2str(host->health_default_recipient); int n_warn=0, n_crit=0; RRDCALC *rc; @@ -330,13 +323,16 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE); crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE); - for(rc = host->alarms; rc && (n_warn + n_crit) < ACTIVE_ALARMS_LIST_EXAMINE ; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; + if(unlikely((n_warn + n_crit) >= ACTIVE_ALARMS_LIST_EXAMINE)) + break; + if (unlikely(rc->status == RRDCALC_STATUS_WARNING)) { if (likely(ae->alarm_id != rc->id) || likely(ae->alarm_event_id != rc->next_event_id - 1)) { - active_alerts[n_warn+n_crit].name = rc->name; + active_alerts[n_warn+n_crit].name = (char *)rrdcalc_name(rc); active_alerts[n_warn+n_crit].last_status_change = rc->last_status_change; active_alerts[n_warn+n_crit].status = rc->status; n_warn++; @@ -344,7 +340,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { expr = rc->warning; } else if (unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) { if (likely(ae->alarm_id != rc->id) || likely(ae->alarm_event_id != rc->next_event_id - 1)) { - active_alerts[n_warn+n_crit].name = rc->name; + active_alerts[n_warn+n_crit].name = (char *)rrdcalc_name(rc); active_alerts[n_warn+n_crit].last_status_change = rc->last_status_change; active_alerts[n_warn+n_crit].status = rc->status; n_crit++; @@ -379,39 +375,39 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { } } - char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN"); + char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN"); snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" NETDATA_DOUBLE_FORMAT_ZERO "' '" NETDATA_DOUBLE_FORMAT_ZERO "' '%s' '%u' '%u' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d' '%s' '%s' '%s' '%s' '%s'", exec, recipient, - host->registry_hostname, + rrdhost_registry_hostname(host), ae->unique_id, ae->alarm_id, ae->alarm_event_id, (unsigned long)ae->when, - ae->name, - ae->chart?ae->chart:"NOCHART", - ae->family?ae->family:"NOFAMILY", + ae_name(ae), + ae->chart?ae_chart_name(ae):"NOCHART", + ae->family?ae_family(ae):"NOFAMILY", rrdcalc_status2string(ae->new_status), rrdcalc_status2string(ae->old_status), ae->new_value, ae->old_value, - ae->source?ae->source:"UNKNOWN", + ae->source?ae_source(ae):"UNKNOWN", (uint32_t)ae->duration, (uint32_t)ae->non_clear_duration, - ae->units?ae->units:"", - ae->info?ae->info:"", - ae->new_value_string, - ae->old_value_string, + ae_units(ae), + ae_info(ae), + ae_new_value_string(ae), + ae_old_value_string(ae), (expr && expr->source)?expr->source:"NOSOURCE", (expr && expr->error_msg)?buffer_tostring(expr->error_msg):"NOERRMSG", n_warn, n_crit, buffer_tostring(warn_alarms), buffer_tostring(crit_alarms), - ae->classification?ae->classification:"Unknown", + ae->classification?ae_classification(ae):"Unknown", edit_command, host != localhost ? host->machine_guid:"" ); @@ -450,7 +446,7 @@ static inline void health_alarm_wait_for_execution(ALARM_ENTRY *ae) { static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) { debug(D_HEALTH, "Health alarm '%s.%s' = " NETDATA_DOUBLE_FORMAT_AUTO " - changed status from %s to %s", - ae->chart?ae->chart:"NOCHART", ae->name, + ae->chart?ae_chart_name(ae):"NOCHART", ae_name(ae), ae->new_value, rrdcalc_status2string(ae->old_status), rrdcalc_status2string(ae->new_status) @@ -467,7 +463,7 @@ static inline void health_alarm_log_process(RRDHOST *host) { ALARM_ENTRY *ae; for(ae = host->health_log.alarms; ae && ae->unique_id >= host->health_last_processed_id; ae = ae->next) { - if(likely(!alarm_entry_isrepeating(host, ae))) { + if(likely(!(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING))) { if(unlikely( !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) && !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED) @@ -508,7 +504,7 @@ static inline void health_alarm_log_process(RRDHOST *host) { ALARM_ENTRY *t = ae->next; - if(likely(!alarm_entry_isrepeating(host, ae))) { + if(likely(!(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING))) { health_alarm_wait_for_execution(ae); health_alarm_log_free_one_nochecks_nounlink(ae); host->health_log.count--; @@ -522,7 +518,7 @@ static inline void health_alarm_log_process(RRDHOST *host) { static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) { if(unlikely(!rc->rrdset)) { - debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name); + debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return 0; } @@ -533,27 +529,27 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) *next_run = rc->next_update; } - debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now)); + debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rrdcalc_chart_name(rc), rrdcalc_name(rc), (int) (rc->next_update - now)); return 0; } if(unlikely(!rc->update_every)) { - debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rc->chart?rc->chart:"NOCHART", rc->name); + debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return 0; } if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE))) { - debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as obsolete", rc->chart?rc->chart:"NOCHART", rc->name); + debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as obsolete", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return 0; } if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_ARCHIVED))) { - debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as archived", rc->chart?rc->chart:"NOCHART", rc->name); + debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as archived", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return 0; } if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) { - debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name); + debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return 0; } @@ -566,7 +562,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) if(unlikely(now + update_every < first /* || now - update_every > last */)) { debug(D_HEALTH , "Health not examining alarm '%s.%s' yet (wanted time is out of bounds - we need %lu but got %lu - %lu)." - , rc->chart ? rc->chart : "NOCHART", rc->name, (unsigned long) now, (unsigned long) first + , rrdcalc_chart_name(rc), rrdcalc_name(rc), (unsigned long) now, (unsigned long) first , (unsigned long) last); return 0; } @@ -577,7 +573,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) if(needed + update_every < first || needed - update_every > last) { debug(D_HEALTH , "Health not examining alarm '%s.%s' yet (not enough data yet - we need %lu but got %lu - %lu)." - , rc->chart ? rc->chart : "NOCHART", rc->name, (unsigned long) needed, (unsigned long) first + , rrdcalc_chart_name(rc), rrdcalc_name(rc), (unsigned long) needed, (unsigned long) first , (unsigned long) last); return 0; } @@ -614,30 +610,30 @@ static void health_main_cleanup(void *ptr) { static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } -static SILENCE_TYPE check_silenced(RRDCALC *rc, char* host, SILENCERS *silencers) { +static SILENCE_TYPE check_silenced(RRDCALC *rc, const char *host, SILENCERS *silencers) { SILENCER *s; debug(D_HEALTH, "Checking if alarm was silenced via the command API. Alarm info name:%s context:%s chart:%s host:%s family:%s", - rc->name, (rc->rrdset)?rc->rrdset->context:"", rc->chart, host, (rc->rrdset)?rc->rrdset->family:""); + rrdcalc_name(rc), (rc->rrdset)?rrdset_context(rc->rrdset):"", rrdcalc_chart_name(rc), host, (rc->rrdset)?rrdset_family(rc->rrdset):""); for (s = silencers->silencers; s!=NULL; s=s->next){ if ( - (!s->alarms_pattern || (rc->name && s->alarms_pattern && simple_pattern_matches(s->alarms_pattern,rc->name))) && - (!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches(s->contexts_pattern,rc->rrdset->context))) && + (!s->alarms_pattern || (rc->name && s->alarms_pattern && simple_pattern_matches(s->alarms_pattern, rrdcalc_name(rc)))) && + (!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches(s->contexts_pattern, rrdset_context(rc->rrdset)))) && (!s->hosts_pattern || (host && s->hosts_pattern && simple_pattern_matches(s->hosts_pattern,host))) && - (!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches(s->charts_pattern,rc->chart))) && - (!s->families_pattern || (rc->rrdset && rc->rrdset->family && s->families_pattern && simple_pattern_matches(s->families_pattern,rc->rrdset->family))) + (!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches(s->charts_pattern, rrdcalc_chart_name(rc)))) && + (!s->families_pattern || (rc->rrdset && rc->rrdset->family && s->families_pattern && simple_pattern_matches(s->families_pattern, rrdset_family(rc->rrdset)))) ) { debug(D_HEALTH, "Alarm matches command API silence entry %s:%s:%s:%s:%s", s->alarms,s->charts, s->contexts, s->hosts, s->families); if (unlikely(silencers->stype == STYPE_NONE)) { - debug(D_HEALTH, "Alarm %s matched a silence entry, but no SILENCE or DISABLE command was issued via the command API. The match has no effect.", rc->name); + debug(D_HEALTH, "Alarm %s matched a silence entry, but no SILENCE or DISABLE command was issued via the command API. The match has no effect.", rrdcalc_name(rc)); } else { debug(D_HEALTH, "Alarm %s via the command API - name:%s context:%s chart:%s host:%s family:%s" , (silencers->stype == STYPE_DISABLE_ALARMS)?"Disabled":"Silenced" - , rc->name - , (rc->rrdset)?rc->rrdset->context:"" - , rc->chart + , rrdcalc_name(rc) + , (rc->rrdset)?rrdset_context(rc->rrdset):"" + , rrdcalc_chart_name(rc) , host - , (rc->rrdset)?rc->rrdset->family:"" + , (rc->rrdset)?rrdset_family(rc->rrdset):"" ); } return silencers->stype; @@ -664,15 +660,15 @@ static int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) { if (silencers->stype == STYPE_DISABLE_ALARMS) rc->rrdcalc_flags |= RRDCALC_FLAG_DISABLED; else if (silencers->stype == STYPE_SILENCE_NOTIFICATIONS) rc->rrdcalc_flags |= RRDCALC_FLAG_SILENCED; } else { - SILENCE_TYPE st = check_silenced(rc, host->hostname, silencers); + SILENCE_TYPE st = check_silenced(rc, rrdhost_hostname(host), silencers); if (st == STYPE_DISABLE_ALARMS) rc->rrdcalc_flags |= RRDCALC_FLAG_DISABLED; else if (st == STYPE_SILENCE_NOTIFICATIONS) rc->rrdcalc_flags |= RRDCALC_FLAG_SILENCED; } if (rrdcalc_flags_old != rc->rrdcalc_flags) { info("Alarm silencing changed for host '%s' alarm '%s': Disabled %s->%s Silenced %s->%s", - host->hostname, - rc->name, + rrdhost_hostname(host), + rrdcalc_name(rc), (rrdcalc_flags_old & RRDCALC_FLAG_DISABLED)?"true":"false", (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false", (rrdcalc_flags_old & RRDCALC_FLAG_SILENCED)?"true":"false", @@ -812,7 +808,7 @@ void *health_main(void *ptr) { info( "Postponing health checks for %"PRId64" seconds, on host '%s'.", (int64_t)hibernation_delay, - host->hostname); + rrdhost_hostname(host)); host->health_delay_up_to = now + hibernation_delay; } @@ -821,7 +817,7 @@ void *health_main(void *ptr) { if (unlikely(now < host->health_delay_up_to)) continue; - info("Resuming health checks on host '%s'.", host->hostname); + info("Resuming health checks on host '%s'.", rrdhost_hostname(host)); host->health_delay_up_to = 0; } @@ -839,7 +835,7 @@ void *health_main(void *ptr) { rrdhost_rdlock(host); // the first loop is to lookup values from the db - for (rc = host->alarms; rc; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if (update_disabled_silenced(host, rc)) continue; @@ -852,10 +848,33 @@ void *health_main(void *ptr) { if (!rrdcalc_isrepeating(rc)) { worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY); time_t now = now_realtime_sec(); + ALARM_ENTRY *ae = health_create_alarm_entry( - host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id, rc->rrdset->context, - rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change, - rc->value, NAN, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info, 0, 0); + host, + rc->id, + rc->next_event_id++, + rc->config_hash_id, + now, + rc->name, + rc->rrdset->id, + rc->rrdset->context, + rc->rrdset->family, + rc->classification, + rc->component, + rc->type, + rc->exec, + rc->recipient, + now - rc->last_status_change, + rc->value, + NAN, + rc->status, + RRDCALC_STATUS_REMOVED, + rc->source, + rc->units, + rc->info, + 0, + rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0); + if (ae) { health_alarm_log(host, ae); rc->old_status = rc->status; @@ -890,7 +909,7 @@ void *health_main(void *ptr) { /* time_t old_db_timestamp = rc->db_before; */ int value_is_null = 0; - int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rc->dimensions, 1, + int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1, rc->after, rc->before, rc->group, NULL, 0, rc->options, &rc->db_after,&rc->db_before, @@ -903,7 +922,7 @@ void *health_main(void *ptr) { rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d", - host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, ret + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret ); } else rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR; @@ -930,13 +949,13 @@ void *health_main(void *ptr) { debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)", - host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc) ); } else rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN; - debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT, host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, - rc->value + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT, + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value ); } @@ -952,7 +971,7 @@ void *health_main(void *ptr) { rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s", - host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg) ); } else { @@ -960,9 +979,9 @@ void *health_main(void *ptr) { debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value " NETDATA_DOUBLE_FORMAT - ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->calculation->parsed_as, rc->calculation->result, - buffer_tostring(rc->calculation->error_msg), rc->source + buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc) ); rc->value = rc->calculation->result; @@ -980,7 +999,7 @@ void *health_main(void *ptr) { if (unlikely(runnable && !netdata_exit)) { rrdhost_rdlock(host); - for (rc = host->alarms; rc; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))) continue; @@ -1002,15 +1021,15 @@ void *health_main(void *ptr) { debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s", - host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), buffer_tostring(rc->warning->error_msg) ); } else { rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value " NETDATA_DOUBLE_FORMAT - ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", - rc->name, rc->warning->result, buffer_tostring(rc->warning->error_msg), rc->source + ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), + rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc) ); warning_status = rrdcalc_value2status(rc->warning->result); } @@ -1028,16 +1047,16 @@ void *health_main(void *ptr) { debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s", - host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), buffer_tostring(rc->critical->error_msg) ); } else { rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value " NETDATA_DOUBLE_FORMAT - ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", - rc->name, rc->critical->result, buffer_tostring(rc->critical->error_msg), - rc->source + ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), + rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg), + rrdcalc_source(rc) ); critical_status = rrdcalc_value2status(rc->critical->result); } @@ -1113,15 +1132,36 @@ void *health_main(void *ptr) { ALARM_ENTRY *ae = health_create_alarm_entry( - host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id, rc->rrdset->context, - rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change, - rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info, - rc->delay_last, - ( - ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | - ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) + host, + rc->id, + rc->next_event_id++, + rc->config_hash_id, + now, + rc->name, + rc->rrdset->id, + rc->rrdset->context, + rc->rrdset->family, + rc->classification, + rc->component, + rc->type, + rc->exec, + rc->recipient, + now - rc->last_status_change, + rc->old_value, + rc->value, + rc->status, + status, + rc->source, + rc->units, + rc->info, + rc->delay_last, + ( + ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | + ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) | + (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0) ) ); + health_alarm_log(host, ae); rc->last_status_change = now; @@ -1138,7 +1178,7 @@ void *health_main(void *ptr) { // process repeating alarms RRDCALC *rc; - for(rc = host->alarms; rc ; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { int repeat_every = 0; if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) { if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) { @@ -1164,16 +1204,38 @@ void *health_main(void *ptr) { worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY); rc->last_repeat = now; if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++; + ALARM_ENTRY *ae = health_create_alarm_entry( - host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id, rc->rrdset->context, - rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change, - rc->old_value, rc->value, rc->old_status, rc->status, rc->source, rc->units, rc->info, - rc->delay_last, - ( - ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | - ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) + host, + rc->id, + rc->next_event_id++, + rc->config_hash_id, + now, + rc->name, + rc->rrdset->id, + rc->rrdset->context, + rc->rrdset->family, + rc->classification, + rc->component, + rc->type, + rc->exec, + rc->recipient, + now - rc->last_status_change, + rc->old_value, + rc->value, + rc->old_status, + rc->status, + rc->source, + rc->units, + rc->info, + rc->delay_last, + ( + ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | + ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) | + (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0) ) ); + ae->last_repeat = rc->last_repeat; if (!(rc->rrdcalc_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) { ae->flags |= HEALTH_ENTRY_RUN_ONCE; diff --git a/health/health.h b/health/health.h index 3e77c12a70..aae608b522 100644 --- a/health/health.h +++ b/health/health.h @@ -14,6 +14,7 @@ extern unsigned int default_health_enabled; #define HEALTH_ENTRY_FLAG_SILENCED 0x00000010 #define HEALTH_ENTRY_RUN_ONCE 0x00000020 #define HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS 0x00000040 +#define HEALTH_ENTRY_FLAG_IS_REPEATING 0x00000080 #define HEALTH_ENTRY_FLAG_SAVED 0x10000000 #define HEALTH_ENTRY_FLAG_ACLK_QUEUED 0x20000000 @@ -35,7 +36,6 @@ extern void health_init(void); extern void health_reload(void); -extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, NETDATA_DOUBLE *result); extern void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* context, RRDCALC_STATUS status); extern void health_alarms2json(RRDHOST *host, BUFFER *wb, int all); extern void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all); @@ -49,30 +49,30 @@ extern void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae); extern void health_alarm_log_load(RRDHOST *host); extern ALARM_ENTRY* health_create_alarm_entry( - RRDHOST *host, - uint32_t alarm_id, - uint32_t alarm_event_id, - uuid_t config_hash_id, - time_t when, - const char *name, - const char *chart, - const char *chart_context, - const char *family, - const char *classification, - const char *component, - const char *type, - const char *exec, - const char *recipient, - time_t duration, - NETDATA_DOUBLE old_value, - NETDATA_DOUBLE new_value, - RRDCALC_STATUS old_status, - RRDCALC_STATUS new_status, - const char *source, - const char *units, - const char *info, - int delay, - uint32_t flags); + RRDHOST *host, + uint32_t alarm_id, + uint32_t alarm_event_id, + const uuid_t config_hash_id, + time_t when, + STRING *name, + STRING *chart, + STRING *chart_context, + STRING *family, + STRING *classification, + STRING *component, + STRING *type, + STRING *exec, + STRING *recipient, + time_t duration, + NETDATA_DOUBLE old_value, + NETDATA_DOUBLE new_value, + RRDCALC_STATUS old_status, + RRDCALC_STATUS new_status, + STRING *source, + STRING *units, + STRING *info, + int delay, + uint32_t flags); extern void health_alarm_log(RRDHOST *host, ALARM_ENTRY *ae); @@ -90,6 +90,6 @@ extern void health_label_log_save(RRDHOST *host); extern char *health_edit_command_from_source(const char *source); extern void sql_refresh_hashes(void); -extern SIMPLE_PATTERN *health_pattern_from_foreach(char *s); +extern SIMPLE_PATTERN *health_pattern_from_foreach(const char *s); #endif //NETDATA_HEALTH_H diff --git a/health/health_config.c b/health/health_config.c index 77b12fc183..eb09a5f81d 100644 --- a/health/health_config.c +++ b/health/health_config.c @@ -35,21 +35,21 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) { if(!rc->chart) { - error("Health configuration for alarm '%s' does not have a chart", rc->name); + error("Health configuration for alarm '%s' does not have a chart", rrdcalc_name(rc)); return 0; } if(!rc->update_every) { - error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rc->chart?rc->chart:"NOCHART", rc->name); + error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return 0; } if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->calculation && !rc->warning && !rc->critical) { - error("Health configuration for alarm '%s.%s' is useless (no db lookup, no calculation, no warning and no critical expressions)", rc->chart?rc->chart:"NOCHART", rc->name); + error("Health configuration for alarm '%s.%s' is useless (no db lookup, no calculation, no warning and no critical expressions)", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return 0; } - if (rrdcalc_exists(host, rc->chart, rc->name, rc->hash_chart, rc->hash)) + if (rrdcalc_exists(host, rrdcalc_chart_name(rc), rrdcalc_name(rc))) return 0; rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id); @@ -57,24 +57,24 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) { debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO ", red " NETDATA_DOUBLE_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", - rc->chart?rc->chart:"NOCHART", - rc->name, + rrdcalc_chart_name(rc), + rrdcalc_name(rc), rc->id, - (rc->exec)?rc->exec:"DEFAULT", - (rc->recipient)?rc->recipient:"DEFAULT", + (rc->exec)?rrdcalc_exec(rc):"DEFAULT", + (rc->recipient)?rrdcalc_recipient(rc):"DEFAULT", rc->green, rc->red, (int)rc->group, rc->after, rc->before, rc->options, - (rc->dimensions)?rc->dimensions:"NONE", - (rc->foreachdim)?rc->foreachdim:"NONE", + (rc->dimensions)?rrdcalc_dimensions(rc):"NONE", + (rc->foreachdim)?rrdcalc_foreachdim(rc):"NONE", rc->update_every, (rc->calculation)?rc->calculation->parsed_as:"NONE", (rc->warning)?rc->warning->parsed_as:"NONE", (rc->critical)?rc->critical->parsed_as:"NONE", - rc->source, + rrdcalc_source(rc), rc->delay_up_duration, rc->delay_down_duration, rc->delay_max_duration, @@ -90,79 +90,53 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) { static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) { if(unlikely(!rt->context)) { - error("Health configuration for template '%s' does not have a context", rt->name); + error("Health configuration for template '%s' does not have a context", rrdcalctemplate_name(rt)); return 0; } if(unlikely(!rt->update_every)) { - error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rt->name); + error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rrdcalctemplate_name(rt)); return 0; } if(unlikely(!RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) && !rt->calculation && !rt->warning && !rt->critical)) { - error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rt->name); + error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rrdcalctemplate_name(rt)); return 0; } - RRDCALCTEMPLATE *t, *last = NULL; - if(!rt->foreachdim) { - for (t = host->templates; t ; last = t, t = t->next) { - if(unlikely(t->hash_name == rt->hash_name - && !strcmp(t->name, rt->name) - && !strcmp(t->family_match?t->family_match:"*", rt->family_match?rt->family_match:"*") - )) { - info("Health configuration template '%s' already exists for host '%s'.", rt->name, host->hostname); - return 0; - } - } - - if(likely(last)) { - last->next = rt; - } - else { - rt->next = host->templates; - host->templates = rt; - } - } else { - for (t = host->alarms_template_with_foreach; t ; last = t, t = t->next) { - if(unlikely(t->hash_name == rt->hash_name - && !strcmp(t->name, rt->name) - && !strcmp(t->family_match?t->family_match:"*", rt->family_match?rt->family_match:"*") - )) { - info("Health configuration template '%s' already exists for host '%s'.", rt->name, host->hostname); - return 0; - } - } - - if(likely(last)) { - last->next = rt; - } - else { - rt->next = host->alarms_template_with_foreach; - host->alarms_template_with_foreach = rt; + RRDCALCTEMPLATE *t; + foreach_rrdcalctemplate_in_rrdhost(host, t) { + if(unlikely(t->name == rt->name && !strcmp(t->family_match?rrdcalctemplate_family_match(t):"*", rt->family_match?rrdcalctemplate_family_match(rt):"*"))) { + info("Health configuration template '%s' already exists for host '%s'.", rrdcalctemplate_name(rt), rrdhost_hostname(host)); + return 0; } } + if(rt->foreachdim) + DOUBLE_LINKED_LIST_PREPEND_UNSAFE(host->alarms_templates, rt, prev, next); + else + DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->alarms_templates, rt, prev, next); + debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO ", red " NETDATA_DOUBLE_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", - rt->name, - (rt->context)?rt->context:"NONE", - (rt->exec)?rt->exec:"DEFAULT", - (rt->recipient)?rt->recipient:"DEFAULT", + rrdcalctemplate_name(rt), + (rt->context)?string2str(rt->context):"NONE", + (rt->exec)?rrdcalctemplate_exec(rt):"DEFAULT", + (rt->recipient)?rrdcalctemplate_recipient(rt):"DEFAULT", rt->green, rt->red, (int)rt->group, rt->after, rt->before, rt->options, - (rt->dimensions)?rt->dimensions:"NONE", - (rt->foreachdim)?rt->foreachdim:"NONE", + (rt->dimensions)?rrdcalctemplate_dimensions(rt):"NONE", + (rt->foreachdim)?rrdcalctemplate_foreachdim(rt):"NONE", rt->update_every, (rt->calculation)?rt->calculation->parsed_as:"NONE", (rt->warning)?rt->warning->parsed_as:"NONE", (rt->critical)?rt->critical->parsed_as:"NONE", - rt->source, + rrdcalctemplate_source(rt), rt->delay_up_duration, rt->delay_down_duration, rt->delay_max_duration, @@ -334,7 +308,7 @@ static inline int health_parse_repeat( * * @param s the string that will be used to create the simple pattern. */ -SIMPLE_PATTERN *health_pattern_from_foreach(char *s) { +SIMPLE_PATTERN *health_pattern_from_foreach(const char *s) { char *convert= strdupz(s); SIMPLE_PATTERN *val = NULL; if(convert) { @@ -350,12 +324,12 @@ SIMPLE_PATTERN *health_pattern_from_foreach(char *s) { static inline int health_parse_db_lookup( size_t line, const char *filename, char *string, RRDR_GROUPING *group_method, int *after, int *before, int *every, - uint32_t *options, char **dimensions, char **foreachdim + uint32_t *options, STRING **dimensions, STRING **foreachdim ) { debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s: %s", line, filename, string); - if(*dimensions) freez(*dimensions); - if(*foreachdim) freez(*foreachdim); + if(*dimensions) string_freez(*dimensions); + if(*foreachdim) string_freez(*foreachdim); *dimensions = NULL; *foreachdim = NULL; *after = 0; @@ -453,7 +427,7 @@ static inline int health_parse_db_lookup( if(find) { *find = '\0'; } - *dimensions = strdupz(s); + *dimensions = string_strdupz(s); } if(!find) { @@ -462,7 +436,7 @@ static inline int health_parse_db_lookup( s = ++find; } else if(!strcasecmp(key, HEALTH_FOREACH_KEY )) { - *foreachdim = strdupz(s); + *foreachdim = string_strdupz(s); break; } else { @@ -474,10 +448,10 @@ static inline int health_parse_db_lookup( return 1; } -static inline char *health_source_file(size_t line, const char *file) { +static inline STRING *health_source_file(size_t line, const char *file) { char buffer[FILENAME_MAX + 1]; snprintfz(buffer, FILENAME_MAX, "%zu@%s", line, file); - return strdupz(buffer); + return string_strdupz(buffer); } char *health_edit_command_from_source(const char *source) @@ -496,7 +470,7 @@ char *health_edit_command_from_source(const char *source) netdata_configured_user_config_dir, file_no_path + 1, temp, - localhost->registry_hostname); + rrdhost_registry_hostname(localhost)); } else buffer[0] = '\0'; @@ -513,35 +487,35 @@ static inline void strip_quotes(char *s) { static inline void alert_config_free(struct alert_config *cfg) { - freez(cfg->alarm); - freez(cfg->template_key); - freez(cfg->os); - freez(cfg->host); - freez(cfg->on); - freez(cfg->families); - freez(cfg->plugin); - freez(cfg->module); - freez(cfg->charts); - freez(cfg->lookup); - freez(cfg->calc); - freez(cfg->warn); - freez(cfg->crit); - freez(cfg->every); - freez(cfg->green); - freez(cfg->red); - freez(cfg->exec); - freez(cfg->to); - freez(cfg->units); - freez(cfg->info); - freez(cfg->classification); - freez(cfg->component); - freez(cfg->type); - freez(cfg->delay); - freez(cfg->options); - freez(cfg->repeat); - freez(cfg->host_labels); - freez(cfg->p_db_lookup_dimensions); - freez(cfg->p_db_lookup_method); + string_freez(cfg->alarm); + string_freez(cfg->template_key); + string_freez(cfg->os); + string_freez(cfg->host); + string_freez(cfg->on); + string_freez(cfg->families); + string_freez(cfg->plugin); + string_freez(cfg->module); + string_freez(cfg->charts); + string_freez(cfg->lookup); + string_freez(cfg->calc); + string_freez(cfg->warn); + string_freez(cfg->crit); + string_freez(cfg->every); + string_freez(cfg->green); + string_freez(cfg->red); + string_freez(cfg->exec); + string_freez(cfg->to); + string_freez(cfg->units); + string_freez(cfg->info); + string_freez(cfg->classification); + string_freez(cfg->component); + string_freez(cfg->type); + string_freez(cfg->delay); + string_freez(cfg->options); + string_freez(cfg->repeat); + string_freez(cfg->host_labels); + string_freez(cfg->p_db_lookup_dimensions); + string_freez(cfg->p_db_lookup_method); freez(cfg); } @@ -685,7 +659,16 @@ static int health_readfile(const char *filename, void *data) { rc = callocz(1, sizeof(RRDCALC)); rc->next_event_id = 1; - rc->name = strdupz(value); + + { + char *tmp = strdupz(value); + if(rrdvar_fix_name(tmp)) + error("Health configuration renamed alarm '%s' to '%s'", value, tmp); + + rc->name = string_strdupz(tmp); + freez(tmp); + } + rc->source = health_source_file(line, filename); rc->green = NAN; rc->red = NAN; @@ -699,59 +682,58 @@ static int health_readfile(const char *filename, void *data) { alert_config_free(alert_cfg); alert_cfg = callocz(1, sizeof(struct alert_config)); - if(rrdvar_fix_name(rc->name)) - error("Health configuration renamed alarm '%s' to '%s'", value, rc->name); - - rc->hash = simple_hash(rc->name); - alert_cfg->alarm = strdupz(rc->name); + alert_cfg->alarm = string_dup(rc->name); ignore_this = 0; } else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) { if(rc) { // health_add_alarms_loop(host, rc, ignore_this) ; - if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) { + if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) rrdcalc_free(rc); - } rc = NULL; } if(rt) { - if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) { + if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) rrdcalctemplate_free(rt); - } } rt = callocz(1, sizeof(RRDCALCTEMPLATE)); - rt->name = strdupz(value); + + { + char *tmp = strdupz(value); + if(rrdvar_fix_name(tmp)) + error("Health configuration renamed template '%s' to '%s'", value, tmp); + + rt->name = string_strdupz(tmp); + freez(tmp); + } + rt->source = health_source_file(line, filename); rt->green = NAN; rt->red = NAN; - rt->delay_multiplier = 1.0; + rt->delay_multiplier = (float)1.0; rt->warn_repeat_every = host->health_default_warn_repeat_every; rt->crit_repeat_every = host->health_default_crit_repeat_every; if (alert_cfg) alert_config_free(alert_cfg); alert_cfg = callocz(1, sizeof(struct alert_config)); - if(rrdvar_fix_name(rt->name)) - error("Health configuration renamed template '%s' to '%s'", value, rt->name); - - rt->hash_name = simple_hash(rt->name); - alert_cfg->template_key = strdupz(rt->name); + alert_cfg->template_key = string_dup(rt->name); ignore_this = 0; } else if(hash == hash_os && !strcasecmp(key, HEALTH_OS_KEY)) { char *os_match = value; - if (alert_cfg) alert_cfg->os = strdupz(value); + if (alert_cfg) alert_cfg->os = string_strdupz(value); SIMPLE_PATTERN *os_pattern = simple_pattern_create(os_match, NULL, SIMPLE_PATTERN_EXACT); - if(!simple_pattern_matches(os_pattern, host->os)) { + if(!simple_pattern_matches(os_pattern, rrdhost_os(host))) { if(rc) - debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s: host O/S does not match '%s'", host->hostname, rc->name, line, filename, os_match); + debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s: host O/S does not match '%s'", rrdhost_hostname(host), rrdcalc_name(rc), line, filename, os_match); if(rt) - debug(D_HEALTH, "HEALTH on '%s' ignoring template '%s' defined at %zu@%s: host O/S does not match '%s'", host->hostname, rt->name, line, filename, os_match); + debug(D_HEALTH, "HEALTH on '%s' ignoring template '%s' defined at %zu@%s: host O/S does not match '%s'", rrdhost_hostname(host), rrdcalctemplate_name(rt), line, filename, os_match); ignore_this = 1; } @@ -760,15 +742,15 @@ static int health_readfile(const char *filename, void *data) { } else if(hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) { char *host_match = value; - if (alert_cfg) alert_cfg->host = strdupz(value); + if (alert_cfg) alert_cfg->host = string_strdupz(value); SIMPLE_PATTERN *host_pattern = simple_pattern_create(host_match, NULL, SIMPLE_PATTERN_EXACT); - if(!simple_pattern_matches(host_pattern, host->hostname)) { + if(!simple_pattern_matches(host_pattern, rrdhost_hostname(host))) { if(rc) - debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s: hostname does not match '%s'", host->hostname, rc->name, line, filename, host_match); + debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s: hostname does not match '%s'", rrdhost_hostname(host), rrdcalc_name(rc), line, filename, host_match); if(rt) - debug(D_HEALTH, "HEALTH on '%s' ignoring template '%s' defined at %zu@%s: hostname does not match '%s'", host->hostname, rt->name, line, filename, host_match); + debug(D_HEALTH, "HEALTH on '%s' ignoring template '%s' defined at %zu@%s: hostname does not match '%s'", rrdhost_hostname(host), rrdcalctemplate_name(rt), line, filename, host_match); ignore_this = 1; } @@ -777,65 +759,68 @@ static int health_readfile(const char *filename, void *data) { } else if(rc) { if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) { - alert_cfg->on = strdupz(value); + alert_cfg->on = string_strdupz(value); if(rc->chart) { - if(strcmp(rc->chart, value) != 0) + if(strcmp(rrdcalc_chart_name(rc), value) != 0) error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rc->name, key, rc->chart, value, value); + line, filename, rrdcalc_name(rc), key, rrdcalc_chart_name(rc), value, value); - freez(rc->chart); + string_freez(rc->chart); } - rc->chart = strdupz(value); - rc->hash_chart = simple_hash(rc->chart); + rc->chart = string_strdupz(value); } else if(hash == hash_class && !strcasecmp(key, HEALTH_CLASS_KEY)) { - alert_cfg->classification = strdupz(value); - if(rc->classification) { - if(strcmp(rc->classification, value) != 0) - error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rc->name, key, rc->classification, value, value); + strip_quotes(value); - freez(rc->classification); + alert_cfg->classification = string_strdupz(value); + if(rc->classification) { + if(strcmp(rrdcalc_classification(rc), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalc_name(rc), key, rrdcalc_classification(rc), value, value); + + string_freez(rc->classification); } - rc->classification = strdupz(value); - strip_quotes(rc->classification); + rc->classification = string_strdupz(value); } else if(hash == hash_component && !strcasecmp(key, HEALTH_COMPONENT_KEY)) { - alert_cfg->component = strdupz(value); - if(rc->component) { - if(strcmp(rc->component, value) != 0) - error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rc->name, key, rc->component, value, value); + strip_quotes(value); - freez(rc->component); + alert_cfg->component = string_strdupz(value); + if(rc->component) { + if(strcmp(rrdcalc_component(rc), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalc_name(rc), key, rrdcalc_component(rc), value, value); + + string_freez(rc->component); } - rc->component = strdupz(value); - strip_quotes(rc->component); + rc->component = string_strdupz(value); } else if(hash == hash_type && !strcasecmp(key, HEALTH_TYPE_KEY)) { - alert_cfg->type = strdupz(value); - if(rc->type) { - if(strcmp(rc->type, value) != 0) - error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rc->name, key, rc->type, value, value); + strip_quotes(value); - freez(rc->type); + alert_cfg->type = string_strdupz(value); + if(rc->type) { + if(strcmp(rrdcalc_type(rc), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalc_name(rc), key, rrdcalc_type(rc), value, value); + + string_freez(rc->type); } - rc->type = strdupz(value); - strip_quotes(rc->type); + rc->type = string_strdupz(value); } else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) { - alert_cfg->lookup = strdupz(value); + alert_cfg->lookup = string_strdupz(value); health_parse_db_lookup(line, filename, value, &rc->group, &rc->after, &rc->before, &rc->update_every, &rc->options, &rc->dimensions, &rc->foreachdim); - if(rc->foreachdim) { - rc->spdim = health_pattern_from_foreach(rc->foreachdim); - } + + if(rc->foreachdim) + rc->spdim = health_pattern_from_foreach(rrdcalc_foreachdim(rc)); + if (rc->after) { if (rc->dimensions) - alert_cfg->p_db_lookup_dimensions = strdupz(rc->dimensions); + alert_cfg->p_db_lookup_dimensions = string_dup(rc->dimensions); if (rc->group) - alert_cfg->p_db_lookup_method = strdupz(group_method2string(rc->group)); + alert_cfg->p_db_lookup_method = string_strdupz(group_method2string(rc->group)); alert_cfg->p_db_lookup_options = rc->options; alert_cfg->p_db_lookup_after = rc->after; alert_cfg->p_db_lookup_before = rc->before; @@ -843,250 +828,261 @@ static int health_readfile(const char *filename, void *data) { } } else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) { - alert_cfg->every = strdupz(value); + alert_cfg->every = string_strdupz(value); if(!config_parse_duration(value, &rc->update_every)) error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.", - line, filename, rc->name, key, value); + line, filename, rrdcalc_name(rc), key, value); alert_cfg->p_update_every = rc->update_every; } else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) { - alert_cfg->green = strdupz(value); + alert_cfg->green = string_strdupz(value); char *e; rc->green = str2ndd(value, &e); if(e && *e) { error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", - line, filename, rc->name, key, e); + line, filename, rrdcalc_name(rc), key, e); } } else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) { - alert_cfg->red = strdupz(value); + alert_cfg->red = string_strdupz(value); char *e; rc->red = str2ndd(value, &e); if(e && *e) { error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", - line, filename, rc->name, key, e); + line, filename, rrdcalc_name(rc), key, e); } } else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) { - alert_cfg->calc = strdupz(value); + alert_cfg->calc = string_strdupz(value); const char *failed_at = NULL; int error = 0; rc->calculation = expression_parse(value, &failed_at, &error); if(!rc->calculation) { error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'", - line, filename, rc->name, key, value, expression_strerror(error), failed_at); + line, filename, rrdcalc_name(rc), key, value, expression_strerror(error), failed_at); } } else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) { - alert_cfg->warn = strdupz(value); + alert_cfg->warn = string_strdupz(value); const char *failed_at = NULL; int error = 0; rc->warning = expression_parse(value, &failed_at, &error); if(!rc->warning) { error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'", - line, filename, rc->name, key, value, expression_strerror(error), failed_at); + line, filename, rrdcalc_name(rc), key, value, expression_strerror(error), failed_at); } } else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) { - alert_cfg->crit = strdupz(value); + alert_cfg->crit = string_strdupz(value); const char *failed_at = NULL; int error = 0; rc->critical = expression_parse(value, &failed_at, &error); if(!rc->critical) { error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'", - line, filename, rc->name, key, value, expression_strerror(error), failed_at); + line, filename, rrdcalc_name(rc), key, value, expression_strerror(error), failed_at); } } else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) { - alert_cfg->exec = strdupz(value); + alert_cfg->exec = string_strdupz(value); if(rc->exec) { - if(strcmp(rc->exec, value) != 0) + if(strcmp(rrdcalc_exec(rc), value) != 0) error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rc->name, key, rc->exec, value, value); + line, filename, rrdcalc_name(rc), key, rrdcalc_exec(rc), value, value); - freez(rc->exec); + string_freez(rc->exec); } - rc->exec = strdupz(value); + rc->exec = string_strdupz(value); } else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) { - alert_cfg->to = strdupz(value); + alert_cfg->to = string_strdupz(value); if(rc->recipient) { - if(strcmp(rc->recipient, value) != 0) + if(strcmp(rrdcalc_recipient(rc), value) != 0) error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rc->name, key, rc->recipient, value, value); + line, filename, rrdcalc_name(rc), key, rrdcalc_recipient(rc), value, value); - freez(rc->recipient); + string_freez(rc->recipient); } - rc->recipient = strdupz(value); + rc->recipient = string_strdupz(value); } else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) { - alert_cfg->units = strdupz(value); - if(rc->units) { - if(strcmp(rc->units, value) != 0) - error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rc->name, key, rc->units, value, value); + strip_quotes(value); - freez(rc->units); + alert_cfg->units = string_strdupz(value); + if(rc->units) { + if(strcmp(rrdcalc_units(rc), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalc_name(rc), key, rrdcalc_units(rc), value, value); + + string_freez(rc->units); } - rc->units = strdupz(value); - strip_quotes(rc->units); + rc->units = string_strdupz(value); } else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) { - alert_cfg->info = strdupz(value); - if(rc->info) { - if(strcmp(rc->info, value) != 0) - error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rc->name, key, rc->info, value, value); + strip_quotes(value); - freez(rc->info); + alert_cfg->info = string_strdupz(value); + if(rc->info) { + if(strcmp(rrdcalc_info(rc), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalc_name(rc), key, rrdcalc_info(rc), value, value); + + string_freez(rc->info); + string_freez(rc->original_info); } - rc->info = strdupz(value); - strip_quotes(rc->info); - rc->original_info = strdupz(value); - strip_quotes(rc->original_info); + rc->info = string_strdupz(value); + rc->original_info = string_dup(rc->info); } else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) { - alert_cfg->delay = strdupz(value); + alert_cfg->delay = string_strdupz(value); health_parse_delay(line, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier); } else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) { - alert_cfg->options = strdupz(value); + alert_cfg->options = string_strdupz(value); rc->options |= health_parse_options(value); } else if(hash == hash_repeat && !strcasecmp(key, HEALTH_REPEAT_KEY)){ - alert_cfg->repeat = strdupz(value); + alert_cfg->repeat = string_strdupz(value); health_parse_repeat(line, filename, value, &rc->warn_repeat_every, &rc->crit_repeat_every); } else if(hash == hash_host_label && !strcasecmp(key, HEALTH_HOST_LABEL_KEY)) { - alert_cfg->host_labels = strdupz(value); + alert_cfg->host_labels = string_strdupz(value); if(rc->host_labels) { - if(strcmp(rc->host_labels, value) != 0) + if(strcmp(rrdcalc_host_labels(rc), value) != 0) error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.", - line, filename, rc->name, key, value, value); + line, filename, rrdcalc_name(rc), key, value, value); - freez(rc->host_labels); + string_freez(rc->host_labels); simple_pattern_free(rc->host_labels_pattern); } - rc->host_labels = simple_pattern_trim_around_equal(value); - rc->host_labels_pattern = simple_pattern_create(rc->host_labels, NULL, SIMPLE_PATTERN_EXACT); + { + char *tmp = simple_pattern_trim_around_equal(value); + rc->host_labels = string_strdupz(tmp); + freez(tmp); + } + rc->host_labels_pattern = simple_pattern_create(rrdcalc_host_labels(rc), NULL, SIMPLE_PATTERN_EXACT); } else if(hash == hash_plugin && !strcasecmp(key, HEALTH_PLUGIN_KEY)) { - alert_cfg->plugin = strdupz(value); - freez(rc->plugin_match); + alert_cfg->plugin = string_strdupz(value); + string_freez(rc->plugin_match); simple_pattern_free(rc->plugin_pattern); - rc->plugin_match = strdupz(value); - rc->plugin_pattern = simple_pattern_create(rc->plugin_match, NULL, SIMPLE_PATTERN_EXACT); + rc->plugin_match = string_strdupz(value); + rc->plugin_pattern = simple_pattern_create(rrdcalc_plugin_match(rc), NULL, SIMPLE_PATTERN_EXACT); } else if(hash == hash_module && !strcasecmp(key, HEALTH_MODULE_KEY)) { - alert_cfg->module = strdupz(value); - freez(rc->module_match); + alert_cfg->module = string_strdupz(value); + string_freez(rc->module_match); simple_pattern_free(rc->module_pattern); - rc->module_match = strdupz(value); - rc->module_pattern = simple_pattern_create(rc->module_match, NULL, SIMPLE_PATTERN_EXACT); + rc->module_match = string_strdupz(value); + rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT); } else { error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.", - line, filename, rc->name, key); + line, filename, rrdcalc_name(rc), key); } } else if(rt) { if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) { - alert_cfg->on = strdupz(value); + alert_cfg->on = string_strdupz(value); if(rt->context) { - if(strcmp(rt->context, value) != 0) + if(strcmp(string2str(rt->context), value) != 0) error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->context, value, value); + line, filename, rrdcalctemplate_name(rt), key, string2str(rt->context), value, value); - freez(rt->context); + string_freez(rt->context); } - rt->context = strdupz(value); - rt->hash_context = simple_hash(rt->context); + rt->context = string_strdupz(value); } else if(hash == hash_class && !strcasecmp(key, HEALTH_CLASS_KEY)) { - alert_cfg->classification = strdupz(value); - if(rt->classification) { - if(strcmp(rt->classification, value) != 0) - error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->classification, value, value); + strip_quotes(value); - freez(rt->classification); + alert_cfg->classification = string_strdupz(value); + if(rt->classification) { + if(strcmp(rrdcalctemplate_classification(rt), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_classification(rt), value, value); + + string_freez(rt->classification); } - rt->classification = strdupz(value); - strip_quotes(rt->classification); + rt->classification = string_strdupz(value); } else if(hash == hash_component && !strcasecmp(key, HEALTH_COMPONENT_KEY)) { - alert_cfg->component = strdupz(value); - if(rt->component) { - if(strcmp(rt->component, value) != 0) - error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->component, value, value); + strip_quotes(value); - freez(rt->component); + alert_cfg->component = string_strdupz(value); + if(rt->component) { + if(strcmp(rrdcalctemplate_component(rt), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_component(rt), value, value); + + string_freez(rt->component); } - rt->component = strdupz(value); - strip_quotes(rt->component); + rt->component = string_strdupz(value); } else if(hash == hash_type && !strcasecmp(key, HEALTH_TYPE_KEY)) { - alert_cfg->type = strdupz(value); - if(rt->type) { - if(strcmp(rt->type, value) != 0) - error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->type, value, value); + strip_quotes(value); - freez(rt->type); + alert_cfg->type = string_strdupz(value); + if(rt->type) { + if(strcmp(rrdcalctemplate_type(rt), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_type(rt), value, value); + + string_freez(rt->type); } - rt->type = strdupz(value); - strip_quotes(rt->type); + rt->type = string_strdupz(value); } else if(hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) { - alert_cfg->families = strdupz(value); - freez(rt->family_match); + alert_cfg->families = string_strdupz(value); + string_freez(rt->family_match); simple_pattern_free(rt->family_pattern); - rt->family_match = strdupz(value); - rt->family_pattern = simple_pattern_create(rt->family_match, NULL, SIMPLE_PATTERN_EXACT); + rt->family_match = string_strdupz(value); + rt->family_pattern = simple_pattern_create(rrdcalctemplate_family_match(rt), NULL, SIMPLE_PATTERN_EXACT); } else if(hash == hash_plugin && !strcasecmp(key, HEALTH_PLUGIN_KEY)) { - alert_cfg->plugin = strdupz(value); - freez(rt->plugin_match); + alert_cfg->plugin = string_strdupz(value); + string_freez(rt->plugin_match); simple_pattern_free(rt->plugin_pattern); - rt->plugin_match = strdupz(value); - rt->plugin_pattern = simple_pattern_create(rt->plugin_match, NULL, SIMPLE_PATTERN_EXACT); + rt->plugin_match = string_strdupz(value); + rt->plugin_pattern = simple_pattern_create(rrdcalctemplate_plugin_match(rt), NULL, SIMPLE_PATTERN_EXACT); } else if(hash == hash_module && !strcasecmp(key, HEALTH_MODULE_KEY)) { - alert_cfg->module = strdupz(value); - freez(rt->module_match); + alert_cfg->module = string_strdupz(value); + string_freez(rt->module_match); simple_pattern_free(rt->module_pattern); - rt->module_match = strdupz(value); - rt->module_pattern = simple_pattern_create(rt->module_match, NULL, SIMPLE_PATTERN_EXACT); + rt->module_match = string_strdupz(value); + rt->module_pattern = simple_pattern_create(rrdcalctemplate_module_match(rt), NULL, SIMPLE_PATTERN_EXACT); } else if(hash == hash_charts && !strcasecmp(key, HEALTH_CHARTS_KEY)) { - alert_cfg->charts = strdupz(value); - freez(rt->charts_match); + alert_cfg->charts = string_strdupz(value); + string_freez(rt->charts_match); simple_pattern_free(rt->charts_pattern); - rt->charts_match = strdupz(value); - rt->charts_pattern = simple_pattern_create(rt->charts_match, NULL, SIMPLE_PATTERN_EXACT); + rt->charts_match = string_strdupz(value); + rt->charts_pattern = simple_pattern_create(rrdcalctemplate_charts_match(rt), NULL, SIMPLE_PATTERN_EXACT); } else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) { - alert_cfg->lookup = strdupz(value); + alert_cfg->lookup = string_strdupz(value); health_parse_db_lookup(line, filename, value, &rt->group, &rt->after, &rt->before, &rt->update_every, &rt->options, &rt->dimensions, &rt->foreachdim); - if(rt->foreachdim) { - rt->spdim = health_pattern_from_foreach(rt->foreachdim); - } + + if(rt->foreachdim) + rt->spdim = health_pattern_from_foreach(rrdcalctemplate_foreachdim(rt)); + if (rt->after) { if (rt->dimensions) - alert_cfg->p_db_lookup_dimensions = strdupz(rt->dimensions); + alert_cfg->p_db_lookup_dimensions = string_dup(rt->dimensions); + if (rt->group) - alert_cfg->p_db_lookup_method = strdupz(group_method2string(rt->group)); + alert_cfg->p_db_lookup_method = string_strdupz(group_method2string(rt->group)); + alert_cfg->p_db_lookup_options = rt->options; alert_cfg->p_db_lookup_after = rt->after; alert_cfg->p_db_lookup_before = rt->before; @@ -1094,137 +1090,143 @@ static int health_readfile(const char *filename, void *data) { } } else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) { - alert_cfg->every = strdupz(value); + alert_cfg->every = string_strdupz(value); if(!config_parse_duration(value, &rt->update_every)) error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' cannot parse duration: '%s'.", - line, filename, rt->name, key, value); + line, filename, rrdcalctemplate_name(rt), key, value); alert_cfg->p_update_every = rt->update_every; } else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) { - alert_cfg->green = strdupz(value); + alert_cfg->green = string_strdupz(value); char *e; rt->green = str2ndd(value, &e); if(e && *e) { error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", - line, filename, rt->name, key, e); + line, filename, rrdcalctemplate_name(rt), key, e); } } else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) { - alert_cfg->red = strdupz(value); + alert_cfg->red = string_strdupz(value); char *e; rt->red = str2ndd(value, &e); if(e && *e) { error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", - line, filename, rt->name, key, e); + line, filename, rrdcalctemplate_name(rt), key, e); } } else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) { - alert_cfg->calc = strdupz(value); + alert_cfg->calc = string_strdupz(value); const char *failed_at = NULL; int error = 0; rt->calculation = expression_parse(value, &failed_at, &error); if(!rt->calculation) { error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'", - line, filename, rt->name, key, value, expression_strerror(error), failed_at); + line, filename, rrdcalctemplate_name(rt), key, value, expression_strerror(error), failed_at); } } else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) { - alert_cfg->warn = strdupz(value); + alert_cfg->warn = string_strdupz(value); const char *failed_at = NULL; int error = 0; rt->warning = expression_parse(value, &failed_at, &error); if(!rt->warning) { error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'", - line, filename, rt->name, key, value, expression_strerror(error), failed_at); + line, filename, rrdcalctemplate_name(rt), key, value, expression_strerror(error), failed_at); } } else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) { - alert_cfg->crit = strdupz(value); + alert_cfg->crit = string_strdupz(value); const char *failed_at = NULL; int error = 0; rt->critical = expression_parse(value, &failed_at, &error); if(!rt->critical) { error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'", - line, filename, rt->name, key, value, expression_strerror(error), failed_at); + line, filename, rrdcalctemplate_name(rt), key, value, expression_strerror(error), failed_at); } } else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) { - alert_cfg->exec = strdupz(value); + alert_cfg->exec = string_strdupz(value); if(rt->exec) { - if(strcmp(rt->exec, value) != 0) + if(strcmp(rrdcalctemplate_exec(rt), value) != 0) error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->exec, value, value); + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_exec(rt), value, value); - freez(rt->exec); + string_freez(rt->exec); } - rt->exec = strdupz(value); + rt->exec = string_strdupz(value); } else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) { - alert_cfg->to = strdupz(value); + alert_cfg->to = string_strdupz(value); if(rt->recipient) { - if(strcmp(rt->recipient, value) != 0) + if(strcmp(rrdcalctemplate_recipient(rt), value) != 0) error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->recipient, value, value); + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_recipient(rt), value, value); - freez(rt->recipient); + string_freez(rt->recipient); } - rt->recipient = strdupz(value); + rt->recipient = string_strdupz(value); } else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) { - alert_cfg->units = strdupz(value); - if(rt->units) { - if(strcmp(rt->units, value) != 0) - error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->units, value, value); + strip_quotes(value); - freez(rt->units); + alert_cfg->units = string_strdupz(value); + if(rt->units) { + if(strcmp(rrdcalctemplate_units(rt), value) != 0) + error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_units(rt), value, value); + + string_freez(rt->units); } - rt->units = strdupz(value); - strip_quotes(rt->units); + rt->units = string_strdupz(value); } else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) { - alert_cfg->info = strdupz(value); - if(rt->info) { - if(strcmp(rt->info, value) != 0) - error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->info, value, value); + strip_quotes(value); - freez(rt->info); + alert_cfg->info = string_strdupz(value); + if(rt->info) { + if(strcmp(rrdcalctemplate_info(rt), value) != 0) + error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_info(rt), value, value); + + string_freez(rt->info); } - rt->info = strdupz(value); - strip_quotes(rt->info); + rt->info = string_strdupz(value); } else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) { - alert_cfg->delay = strdupz(value); + alert_cfg->delay = string_strdupz(value); health_parse_delay(line, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier); } else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) { - alert_cfg->options = strdupz(value); + alert_cfg->options = string_strdupz(value); rt->options |= health_parse_options(value); } else if(hash == hash_repeat && !strcasecmp(key, HEALTH_REPEAT_KEY)){ - alert_cfg->repeat = strdupz(value); + alert_cfg->repeat = string_strdupz(value); health_parse_repeat(line, filename, value, &rt->warn_repeat_every, &rt->crit_repeat_every); } else if(hash == hash_host_label && !strcasecmp(key, HEALTH_HOST_LABEL_KEY)) { - alert_cfg->host_labels = strdupz(value); + alert_cfg->host_labels = string_strdupz(value); if(rt->host_labels) { - if(strcmp(rt->host_labels, value) != 0) + if(strcmp(rrdcalctemplate_host_labels(rt), value) != 0) error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, filename, rt->name, key, rt->host_labels, value, value); + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_host_labels(rt), value, value); - freez(rt->host_labels); + string_freez(rt->host_labels); simple_pattern_free(rt->host_labels_pattern); } - rt->host_labels = simple_pattern_trim_around_equal(value); - rt->host_labels_pattern = simple_pattern_create(rt->host_labels, NULL, SIMPLE_PATTERN_EXACT); + { + char *tmp = simple_pattern_trim_around_equal(value); + rt->host_labels = string_strdupz(tmp); + freez(tmp); + } + rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL, SIMPLE_PATTERN_EXACT); } else { error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.", - line, filename, rt->name, key); + line, filename, rrdcalctemplate_name(rt), key); } } else { @@ -1260,7 +1262,7 @@ void sql_refresh_hashes(void) void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath) { if(unlikely(!host->health_enabled)) { - debug(D_HEALTH, "CONFIG health is not enabled for host '%s'", host->hostname); + debug(D_HEALTH, "CONFIG health is not enabled for host '%s'", rrdhost_hostname(host)); return; } diff --git a/health/health_json.c b/health/health_json.c index a0dc55b612..a19526d344 100644 --- a/health/health_json.c +++ b/health/health_json.c @@ -14,7 +14,7 @@ void health_string2json(BUFFER *wb, const char *prefix, const char *label, const } void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) { - char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN"); + char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN"); char config_hash_id[GUID_LEN + 1]; uuid_unparse_lower(ae->config_hash_id, config_hash_id); @@ -57,30 +57,30 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) "\t\t\"old_value_string\": \"%s\",\n" "\t\t\"last_repeat\": \"%lu\",\n" "\t\t\"silenced\": \"%s\",\n" - , host->hostname + , rrdhost_hostname(host) , host->utc_offset - , host->abbrev_timezone + , rrdhost_abbrev_timezone(host) , ae->unique_id , ae->alarm_id , ae->alarm_event_id , config_hash_id - , ae->name - , ae->chart - , ae->chart_context - , ae->family - , ae->classification?ae->classification:"Unknown" - , ae->component?ae->component:"Unknown" - , ae->type?ae->type:"Unknown" + , ae_name(ae) + , ae_chart_name(ae) + , ae_chart_context(ae) + , ae_family(ae) + , ae->classification?ae_classification(ae):"Unknown" + , ae->component?ae_component(ae):"Unknown" + , ae->type?ae_type(ae):"Unknown" , (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false" , (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false" , (unsigned long)ae->exec_run_timestamp , (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false" - , ae->exec?ae->exec:host->health_default_exec - , ae->recipient?ae->recipient:host->health_default_recipient + , ae->exec?ae_exec(ae):string2str(host->health_default_exec) + , ae->recipient?ae_recipient(ae):string2str(host->health_default_recipient) , ae->exec_code - , ae->source + , ae_source(ae) , edit_command - , ae->units?ae->units:"" + , ae_units(ae) , (unsigned long)ae->when , (unsigned long)ae->duration , (unsigned long)ae->non_clear_duration @@ -90,13 +90,13 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) , (unsigned long)ae->delay_up_to_timestamp , ae->updated_by_id , ae->updates_id - , ae->new_value_string - , ae->old_value_string + , ae_new_value_string(ae) + , ae_old_value_string(ae) , (unsigned long)ae->last_repeat , (ae->flags & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false" ); - health_string2json(wb, "\t\t", "info", ae->info ? ae->info : "", ",\n"); + health_string2json(wb, "\t\t", "info", ae->info ? ae_info(ae) : "", ",\n"); if(unlikely(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) { buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n"); @@ -122,11 +122,12 @@ void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *char unsigned int max = host->health_log.max; unsigned int count = 0; - uint32_t hash_chart = 0; - if (chart) hash_chart = simple_hash(chart); + + STRING *chart_string = string_strdupz(chart); + ALARM_ENTRY *ae; for (ae = host->health_log.alarms; ae && count < max; ae = ae->next) { - if ((ae->unique_id > after) && (!chart || (ae->hash_chart == hash_chart && !strcmp(ae->chart, chart)))) { + if ((ae->unique_id > after) && (!chart || chart_string == ae->chart)) { if (likely(count)) buffer_strcat(wb, ","); health_alarm_entry2json_nolock(wb, ae, host); @@ -134,6 +135,8 @@ void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *char } } + string_freez(chart_string); + buffer_strcat(wb, "\n]\n"); netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); @@ -144,7 +147,7 @@ static inline void health_rrdcalc_values2json_nolock(RRDHOST *host, BUFFER *wb, buffer_sprintf(wb, "\t\t\"%s.%s\": {\n" "\t\t\t\"id\": %lu,\n" - , rc->chart, rc->name + , rrdcalc_chart_name(rc), rrdcalc_name(rc) , (unsigned long)rc->id); buffer_strcat(wb, "\t\t\t\"value\":"); @@ -164,7 +167,7 @@ static inline void health_rrdcalc_values2json_nolock(RRDHOST *host, BUFFER *wb, static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC *rc) { char value_string[100 + 1]; - format_value_and_unit(value_string, 100, rc->value, rc->units, -1); + format_value_and_unit(value_string, 100, rc->value, rrdcalc_units(rc), -1); char hash_id[GUID_LEN + 1]; uuid_unparse_lower(rc->config_hash_id, hash_id); @@ -203,23 +206,23 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC "\t\t\t\"value_string\": \"%s\",\n" "\t\t\t\"last_repeat\": \"%lu\",\n" "\t\t\t\"times_repeat\": %lu,\n" - , rc->chart, rc->name + , rrdcalc_chart_name(rc), rrdcalc_name(rc) , (unsigned long)rc->id , hash_id - , rc->name - , rc->chart - , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:"" - , rc->classification?rc->classification:"Unknown" - , rc->component?rc->component:"Unknown" - , rc->type?rc->type:"Unknown" + , rrdcalc_name(rc) + , rrdcalc_chart_name(rc) + , (rc->rrdset)?rrdset_family(rc->rrdset):"" + , rc->classification?rrdcalc_classification(rc):"Unknown" + , rc->component?rrdcalc_component(rc):"Unknown" + , rc->type?rrdcalc_type(rc):"Unknown" , (rc->rrdset)?"true":"false" , (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false" , (rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)?"true":"false" - , rc->exec?rc->exec:host->health_default_exec - , rc->recipient?rc->recipient:host->health_default_recipient - , rc->source - , rc->units?rc->units:"" - , rc->info?rc->info:"" + , rc->exec?rrdcalc_exec(rc):string2str(host->health_default_exec) + , rc->recipient?rrdcalc_recipient(rc):string2str(host->health_default_recipient) + , rrdcalc_source(rc) + , rrdcalc_units(rc) + , rrdcalc_info(rc) , rrdcalc_status2string(rc->status) , (unsigned long)rc->last_status_change , (unsigned long)rc->last_updated @@ -243,8 +246,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC } if(RRDCALC_HAS_DB_LOOKUP(rc)) { - if(rc->dimensions && *rc->dimensions) - health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n"); + if(rc->dimensions) + health_string2json(wb, "\t\t\t", "lookup_dimensions", rrdcalc_dimensions(rc), ",\n"); buffer_sprintf(wb, "\t\t\t\"db_after\": %lu,\n" @@ -310,20 +313,24 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL while(p && *p && (tok = mystrsep(&p, ", |"))) { if(!*tok) continue; - for(rc = host->alarms; rc ; rc = rc->next) { + STRING *tok_string = string_strdupz(tok); + + foreach_rrdcalc_in_rrdhost(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; if (unlikely(!rrdset_is_available_for_exporting_and_alarms(rc->rrdset))) continue; - if(unlikely(rc->rrdset && rc->rrdset->hash_context == simple_hash(tok) - && !strcmp(rc->rrdset->context, tok) - && ((status==RRDCALC_STATUS_RAISED)?(rc->status >= RRDCALC_STATUS_WARNING):rc->status == status))) + if(unlikely(rc->rrdset + && rc->rrdset->context == tok_string + && ((status==RRDCALC_STATUS_RAISED)?(rc->status >= RRDCALC_STATUS_WARNING):rc->status == status))) numberOfAlarms++; } + + string_freez(tok_string); } } else { - for(rc = host->alarms; rc ; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; if (unlikely(!rrdset_is_available_for_exporting_and_alarms(rc->rrdset))) @@ -339,8 +346,8 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL static void health_alarms2json_fill_alarms(RRDHOST *host, BUFFER *wb, int all, void (*fp)(RRDHOST *, BUFFER *, RRDCALC *)) { RRDCALC *rc; - int i; - for(i = 0, rc = host->alarms; rc ; rc = rc->next) { + int i = 0; + foreach_rrdcalc_in_rrdhost(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; @@ -363,7 +370,7 @@ void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) { "\n\t\"status\": %s," "\n\t\"now\": %lu," "\n\t\"alarms\": {\n", - host->hostname, + rrdhost_hostname(host), (host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0, host->health_enabled?"true":"false", (unsigned long)now_realtime_sec()); @@ -383,7 +390,7 @@ void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all) { rrdhost_rdlock(host); buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\"," "\n\t\"alarms\": {\n", - host->hostname); + rrdhost_hostname(host)); health_alarms2json_fill_alarms(host, wb, all, health_rrdcalc_values2json_nolock); diff --git a/health/health_log.c b/health/health_log.c index d880625e05..c20fb85e65 100644 --- a/health/health_log.c +++ b/health/health_log.c @@ -14,11 +14,11 @@ inline int health_alarm_log_open(RRDHOST *host) { if(host->health_log_fp) { if (setvbuf(host->health_log_fp, NULL, _IOLBF, 0) != 0) - error("HEALTH [%s]: cannot set line buffering on health log file '%s'.", host->hostname, host->health_log_filename); + error("HEALTH [%s]: cannot set line buffering on health log file '%s'.", rrdhost_hostname(host), host->health_log_filename); return 0; } - error("HEALTH [%s]: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", host->hostname, host->health_log_filename); + error("HEALTH [%s]: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", rrdhost_hostname(host), host->health_log_filename); return -1; } @@ -45,13 +45,13 @@ static inline void health_log_rotate(RRDHOST *host) { snprintfz(old_filename, FILENAME_MAX, "%s.old", host->health_log_filename); if(unlink(old_filename) == -1 && errno != ENOENT) - error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, old_filename); + error("HEALTH [%s]: cannot remove old alarms log file '%s'", rrdhost_hostname(host), old_filename); if(link(host->health_log_filename, old_filename) == -1 && errno != ENOENT) - error("HEALTH [%s]: cannot move file '%s' to '%s'.", host->hostname, host->health_log_filename, old_filename); + error("HEALTH [%s]: cannot move file '%s' to '%s'.", rrdhost_hostname(host), host->health_log_filename, old_filename); if(unlink(host->health_log_filename) == -1 && errno != ENOENT) - error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, host->health_log_filename); + error("HEALTH [%s]: cannot remove old alarms log file '%s'", rrdhost_hostname(host), host->health_log_filename); // open it with truncate host->health_log_fp = fopen(host->health_log_filename, "w"); @@ -59,7 +59,7 @@ static inline void health_log_rotate(RRDHOST *host) { if(host->health_log_fp) fclose(host->health_log_fp); else - error("HEALTH [%s]: cannot truncate health log '%s'", host->hostname, host->health_log_filename); + error("HEALTH [%s]: cannot truncate health log '%s'", rrdhost_hostname(host), host->health_log_filename); host->health_log_fp = NULL; @@ -80,7 +80,7 @@ inline void health_label_log_save(RRDHOST *host) { if (unlikely(fprintf(host->health_log_fp, "L\t%s", write) < 0)) error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", - host->hostname, host->health_log_filename); + rrdhost_hostname(host), host->health_log_filename); else host->health_log_entries_written++; @@ -103,7 +103,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) { "\t%s\t%s\t%s" "\n" , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A' - , host->hostname + , rrdhost_hostname(host) , ae->unique_id , ae->alarm_id @@ -118,14 +118,14 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) { , (uint32_t)ae->exec_run_timestamp , (uint32_t)ae->delay_up_to_timestamp - , (ae->name)?ae->name:"" - , (ae->chart)?ae->chart:"" - , (ae->family)?ae->family:"" - , (ae->exec)?ae->exec:"" - , (ae->recipient)?ae->recipient:"" - , (ae->source)?ae->source:"" - , (ae->units)?ae->units:"" - , (ae->info)?ae->info:"" + , ae_name(ae) + , ae_chart_name(ae) + , ae_family(ae) + , ae_exec(ae) + , ae_recipient(ae) + , ae_source(ae) + , ae_units(ae) + , ae_info(ae) , ae->exec_code , ae->new_status @@ -135,11 +135,11 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) { , ae->new_value , ae->old_value , (uint64_t)ae->last_repeat - , (ae->classification)?ae->classification:"Unknown" - , (ae->component)?ae->component:"Unknown" - , (ae->type)?ae->type:"Unknown" + , (ae->classification)?ae_classification(ae):"Unknown" + , (ae->component)?ae_component(ae):"Unknown" + , (ae->type)?ae_type(ae):"Unknown" ) < 0)) - error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", host->hostname, host->health_log_filename); + error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", rrdhost_hostname(host), host->health_log_filename); else { ae->flags |= HEALTH_ENTRY_FLAG_SAVED; host->health_log_entries_written++; @@ -156,18 +156,23 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) { static uint32_t is_valid_alarm_id(RRDHOST *host, const char *chart, const char *name, uint32_t alarm_id) { - uint32_t hash_chart = simple_hash(chart); - uint32_t hash_name = simple_hash(name); + STRING *chart_string = string_strdupz(chart); + STRING *name_string = string_strdupz(name); + + uint32_t ret = 1; ALARM_ENTRY *ae; for(ae = host->health_log.alarms; ae ;ae = ae->next) { - if (unlikely( - ae->alarm_id == alarm_id && (!(ae->hash_name == hash_name && ae->hash_chart == hash_chart && - !strcmp(name, ae->name) && !strcmp(chart, ae->chart))))) { - return 0; + if (unlikely(ae->alarm_id == alarm_id && (!(chart_string == ae->chart && name_string == ae->name)))) { + ret = 0; + break; } } - return 1; + + string_freez(chart_string); + string_freez(name_string); + + return ret; } static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) { @@ -179,6 +184,11 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); + DICTIONARY *all_rrdcalcs = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + RRDCALC *rc; + foreach_rrdcalc_in_rrdhost(host, rc) + dictionary_set(all_rrdcalcs, rrdcalc_name(rc), rc, sizeof(*rc)); + while((s = fgets_trim_len(buf, 65536, fp, &len))) { host->health_log_entries_written++; line++; @@ -192,7 +202,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *s = '\0'; pointers[entries++] = ++s; if(entries >= max_entries) { - error("HEALTH [%s]: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", host->hostname, line, filename, max_entries); + error("HEALTH [%s]: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", rrdhost_hostname(host), line, filename, max_entries); break; } } @@ -206,7 +216,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char ALARM_ENTRY *ae = NULL; if(entries < 27) { - error("HEALTH [%s]: line %zu of file '%s' should have at least 27 entries, but it has %d. Ignoring it.", host->hostname, line, filename, entries); + error("HEALTH [%s]: line %zu of file '%s' should have at least 27 entries, but it has %d. Ignoring it.", rrdhost_hostname(host), line, filename, entries); errored++; continue; } @@ -214,14 +224,14 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char // check that we have valid ids uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16); if(!unique_id) { - error("HEALTH [%s]: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", host->hostname, line, filename, unique_id, pointers[2]); + error("HEALTH [%s]: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", rrdhost_hostname(host), line, filename, unique_id, pointers[2]); errored++; continue; } uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16); if(!alarm_id) { - error("HEALTH [%s]: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", host->hostname, line, filename, alarm_id, pointers[3]); + error("HEALTH [%s]: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", rrdhost_hostname(host), line, filename, alarm_id, pointers[3]); errored++; continue; } @@ -232,18 +242,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char char* alarm_name = pointers[13]; last_repeat = (time_t)strtoul(pointers[27], NULL, 16); - RRDCALC *rc = alarm_max_last_repeat(host, alarm_name,simple_hash(alarm_name)); - if (!rc) { - for(rc = host->alarms; rc ; rc = rc->next) { - RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_name, (avl_t *)rc); - if(rdcmp != rc) { - error("Cannot insert the alarm index ID using log %s", rc->name); - } - } - - rc = alarm_max_last_repeat(host, alarm_name,simple_hash(alarm_name)); - } - + rc = dictionary_get(all_rrdcalcs, alarm_name); if(unlikely(rc)) { if (rrdcalc_isrepeating(rc)) { rc->last_repeat = last_repeat; @@ -259,7 +258,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char // make sure it is properly numbered if(unlikely(host->health_log.alarms && unique_id < host->health_log.alarms->unique_id)) { error( "HEALTH [%s]: line %zu of file '%s' has alarm log entry %u in wrong order. Ignoring it." - , host->hostname, line, filename, unique_id); + , rrdhost_hostname(host), line, filename, unique_id); errored++; continue; } @@ -272,7 +271,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char if(unlikely(unique_id == ae->unique_id)) { if(unlikely(*pointers[0] == 'A')) { error("HEALTH [%s]: line %zu of file '%s' adds duplicate alarm log entry %u. Using the later." - , host->hostname, line, filename, unique_id); + , rrdhost_hostname(host), line, filename, unique_id); *pointers[0] = 'U'; duplicate++; } @@ -298,8 +297,13 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char // error("HEALTH [%s]: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", host->hostname, line, filename, pointers[1], host->hostname); ae->unique_id = unique_id; - if (!is_valid_alarm_id(host, pointers[14], pointers[13], alarm_id)) - alarm_id = rrdcalc_get_unique_id(host, pointers[14], pointers[13], NULL); + if (!is_valid_alarm_id(host, pointers[14], pointers[13], alarm_id)) { + STRING *chart = string_strdupz(pointers[14]); + STRING *name = string_strdupz(pointers[13]); + alarm_id = rrdcalc_get_unique_id(host, chart, name, NULL); + string_freez(chart); + string_freez(name); + } ae->alarm_id = alarm_id; ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16); ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16); @@ -315,36 +319,29 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16); ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16); - freez(ae->name); - ae->name = strdupz(pointers[13]); - ae->hash_name = simple_hash(ae->name); + string_freez(ae->name); + ae->name = string_strdupz(pointers[13]); - freez(ae->chart); - ae->chart = strdupz(pointers[14]); - ae->hash_chart = simple_hash(ae->chart); + string_freez(ae->chart); + ae->chart = string_strdupz(pointers[14]); - freez(ae->family); - ae->family = strdupz(pointers[15]); + string_freez(ae->family); + ae->family = string_strdupz(pointers[15]); - freez(ae->exec); - ae->exec = strdupz(pointers[16]); - if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; } + string_freez(ae->exec); + ae->exec = string_strdupz(pointers[16]); - freez(ae->recipient); - ae->recipient = strdupz(pointers[17]); - if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; } + string_freez(ae->recipient); + ae->recipient = string_strdupz(pointers[17]); - freez(ae->source); - ae->source = strdupz(pointers[18]); - if(!*ae->source) { freez(ae->source); ae->source = NULL; } + string_freez(ae->source); + ae->source = string_strdupz(pointers[18]); - freez(ae->units); - ae->units = strdupz(pointers[19]); - if(!*ae->units) { freez(ae->units); ae->units = NULL; } + string_freez(ae->units); + ae->units = string_strdupz(pointers[19]); - freez(ae->info); - ae->info = strdupz(pointers[20]); - if(!*ae->info) { freez(ae->info); ae->info = NULL; } + string_freez(ae->info); + ae->info = string_strdupz(pointers[20]); ae->exec_code = str2i(pointers[21]); ae->new_status = str2i(pointers[22]); @@ -357,24 +354,21 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char ae->last_repeat = last_repeat; if (likely(entries > 30)) { - freez(ae->classification); - ae->classification = strdupz(pointers[28]); - if(!*ae->classification) { freez(ae->classification); ae->classification = NULL; } + string_freez(ae->classification); + ae->classification = string_strdupz(pointers[28]); - freez(ae->component); - ae->component = strdupz(pointers[29]); - if(!*ae->component) { freez(ae->component); ae->component = NULL; } + string_freez(ae->component); + ae->component = string_strdupz(pointers[29]); - freez(ae->type); - ae->type = strdupz(pointers[30]); - if(!*ae->type) { freez(ae->type); ae->type = NULL; } + string_freez(ae->type); + ae->type = string_strdupz(pointers[30]); } char value_string[100 + 1]; - freez(ae->old_value_string); - freez(ae->new_value_string); - ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1)); - ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1)); + string_freez(ae->old_value_string); + string_freez(ae->new_value_string); + ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1)); + ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1)); // add it to host if not already there if(unlikely(*pointers[0] == 'A')) { @@ -395,11 +389,14 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char host->health_max_alarm_id = ae->alarm_id; } else { - error("HEALTH [%s]: line %zu of file '%s' is invalid (unrecognized entry type '%s').", host->hostname, line, filename, pointers[0]); + error("HEALTH [%s]: line %zu of file '%s' is invalid (unrecognized entry type '%s').", rrdhost_hostname(host), line, filename, pointers[0]); errored++; } } + dictionary_destroy(all_rrdcalcs); + all_rrdcalcs = NULL; + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); freez(buf); @@ -411,7 +408,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char if (unlikely(!host->health_log.next_alarm_id || host->health_log.next_alarm_id <= host->health_max_alarm_id)) host->health_log.next_alarm_id = host->health_max_alarm_id + 1; - debug(D_HEALTH, "HEALTH [%s]: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", host->hostname, filename, loaded, updated, errored, duplicate); + debug(D_HEALTH, "HEALTH [%s]: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", rrdhost_hostname(host), filename, loaded, updated, errored, duplicate); return loaded; } @@ -422,7 +419,7 @@ inline void health_alarm_log_load(RRDHOST *host) { snprintfz(filename, FILENAME_MAX, "%s.old", host->health_log_filename); FILE *fp = fopen(filename, "r"); if(!fp) - error("HEALTH [%s]: cannot open health file: %s", host->hostname, filename); + error("HEALTH [%s]: cannot open health file: %s", rrdhost_hostname(host), filename); else { health_alarm_log_read(host, fp, filename); fclose(fp); @@ -431,7 +428,7 @@ inline void health_alarm_log_load(RRDHOST *host) { host->health_log_entries_written = 0; fp = fopen(host->health_log_filename, "r"); if(!fp) - error("HEALTH [%s]: cannot open health file: %s", host->hostname, host->health_log_filename); + error("HEALTH [%s]: cannot open health file: %s", rrdhost_hostname(host), host->health_log_filename); else { health_alarm_log_read(host, fp, host->health_log_filename); fclose(fp); @@ -443,63 +440,48 @@ inline void health_alarm_log_load(RRDHOST *host) { // health alarm log management inline ALARM_ENTRY* health_create_alarm_entry( - RRDHOST *host, - uint32_t alarm_id, - uint32_t alarm_event_id, - uuid_t config_hash_id, - time_t when, - const char *name, - const char *chart, - const char *chart_context, - const char *family, - const char *class, - const char *component, - const char *type, - const char *exec, - const char *recipient, - time_t duration, - NETDATA_DOUBLE old_value, - NETDATA_DOUBLE new_value, - RRDCALC_STATUS old_status, - RRDCALC_STATUS new_status, - const char *source, - const char *units, - const char *info, - int delay, - uint32_t flags + RRDHOST *host, + uint32_t alarm_id, + uint32_t alarm_event_id, + const uuid_t config_hash_id, + time_t when, + STRING *name, + STRING *chart, + STRING *chart_context, + STRING *family, + STRING *class, + STRING *component, + STRING *type, + STRING *exec, + STRING *recipient, + time_t duration, + NETDATA_DOUBLE old_value, + NETDATA_DOUBLE new_value, + RRDCALC_STATUS old_status, + RRDCALC_STATUS new_status, + STRING *source, + STRING *units, + STRING *info, + int delay, + uint32_t flags ) { debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id); ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY)); - ae->name = strdupz(name); - ae->hash_name = simple_hash(ae->name); - - if(chart) { - ae->chart = strdupz(chart); - ae->hash_chart = simple_hash(ae->chart); - } - - if(chart_context) - ae->chart_context = strdupz(chart_context); + ae->name = string_dup(name); + ae->chart = string_dup(chart); + ae->chart_context = string_dup(chart_context); uuid_copy(ae->config_hash_id, *((uuid_t *) config_hash_id)); - if(family) - ae->family = strdupz(family); - - if (class) - ae->classification = strdupz(class); - - if (component) - ae->component = strdupz(component); - - if (type) - ae->type = strdupz(type); - - if(exec) ae->exec = strdupz(exec); - if(recipient) ae->recipient = strdupz(recipient); - if(source) ae->source = strdupz(source); - if(units) ae->units = strdupz(units); + ae->family = string_dup(family); + ae->classification = string_dup(class); + ae->component = string_dup(component); + ae->type = string_dup(type); + ae->exec = string_dup(exec); + ae->recipient = string_dup(recipient); + ae->source = string_dup(source); + ae->units = string_dup(units); ae->unique_id = host->health_log.next_log_id++; ae->alarm_id = alarm_id; @@ -509,12 +491,10 @@ inline ALARM_ENTRY* health_create_alarm_entry( ae->new_value = new_value; char value_string[100 + 1]; - ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1)); - ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1)); - - if (info) - ae->info = strdupz(info); + ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1)); + ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1)); + ae->info = string_dup(info); ae->old_status = old_status; ae->new_status = new_status; ae->duration = duration; @@ -570,20 +550,20 @@ inline void health_alarm_log( } inline void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae) { - freez(ae->name); - freez(ae->chart); - freez(ae->chart_context); - freez(ae->family); - freez(ae->classification); - freez(ae->component); - freez(ae->type); - freez(ae->exec); - freez(ae->recipient); - freez(ae->source); - freez(ae->units); - freez(ae->info); - freez(ae->old_value_string); - freez(ae->new_value_string); + string_freez(ae->name); + string_freez(ae->chart); + string_freez(ae->chart_context); + string_freez(ae->family); + string_freez(ae->classification); + string_freez(ae->component); + string_freez(ae->type); + string_freez(ae->exec); + string_freez(ae->recipient); + string_freez(ae->source); + string_freez(ae->units); + string_freez(ae->info); + string_freez(ae->old_value_string); + string_freez(ae->new_value_string); freez(ae); } diff --git a/libnetdata/circular_buffer/circular_buffer.c b/libnetdata/circular_buffer/circular_buffer.c index 998008db27..f074996d9d 100644 --- a/libnetdata/circular_buffer/circular_buffer.c +++ b/libnetdata/circular_buffer/circular_buffer.c @@ -46,6 +46,11 @@ static int cbuffer_realloc_unsafe(struct circular_buffer *buf) { return 0; } +size_t cbuffer_available_size_unsafe(struct circular_buffer *buf) { + size_t len = (buf->write >= buf->read) ? (buf->write - buf->read) : (buf->size - buf->read + buf->write); + return buf->max_size - len; +} + int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len) { size_t len = (buf->write >= buf->read) ? (buf->write - buf->read) : (buf->size - buf->read + buf->write); while (d_len + len >= buf->size) { @@ -78,6 +83,7 @@ void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num) { size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start) { if (start != NULL) *start = buf->data + buf->read; + if (buf->read <= buf->write) { return buf->write - buf->read; // Includes empty case } diff --git a/libnetdata/circular_buffer/circular_buffer.h b/libnetdata/circular_buffer/circular_buffer.h index ba37e0ebf9..e5addc50f4 100644 --- a/libnetdata/circular_buffer/circular_buffer.h +++ b/libnetdata/circular_buffer/circular_buffer.h @@ -13,4 +13,6 @@ extern void cbuffer_free(struct circular_buffer *buf); extern int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len); extern void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num); extern size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start); +extern size_t cbuffer_available_size_unsafe(struct circular_buffer *buf); + #endif diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c index c1325ecb54..6e37554ea4 100644 --- a/libnetdata/dictionary/dictionary.c +++ b/libnetdata/dictionary/dictionary.c @@ -1,28 +1,17 @@ // SPDX-License-Identifier: GPL-3.0-or-later // NOT TO BE USED BY USERS -#define DICTIONARY_FLAG_EXCLUSIVE_ACCESS (1 << 29) // there is only one thread accessing the dictionary -#define DICTIONARY_FLAG_DESTROYED (1 << 30) // this dictionary has been destroyed -#define DICTIONARY_FLAG_DEFER_ALL_DELETIONS (1 << 31) // defer all deletions of items in the dictionary +#define DICTIONARY_FLAG_EXCLUSIVE_ACCESS (1 << 28) // there is only one thread accessing the dictionary +#define DICTIONARY_FLAG_DESTROYED (1 << 29) // this dictionary has been destroyed +#define DICTIONARY_FLAG_DEFER_ALL_DELETIONS (1 << 30) // defer all deletions of items in the dictionary // our reserved flags that cannot be set by users #define DICTIONARY_FLAGS_RESERVED (DICTIONARY_FLAG_EXCLUSIVE_ACCESS|DICTIONARY_FLAG_DESTROYED|DICTIONARY_FLAG_DEFER_ALL_DELETIONS) -typedef struct dictionary DICTIONARY; #define DICTIONARY_INTERNALS #include "../libnetdata.h" - -#ifndef ENABLE_DBENGINE -#define DICTIONARY_WITH_AVL -#warning Compiling DICTIONARY with an AVL index -#else -#define DICTIONARY_WITH_JUDYHS -#endif - -#ifdef DICTIONARY_WITH_JUDYHS #include -#endif typedef enum name_value_flags { NAME_VALUE_FLAG_NONE = 0, @@ -38,10 +27,6 @@ typedef enum name_value_flags { */ typedef struct name_value { -#ifdef DICTIONARY_WITH_AVL - avl_t avl_node; -#endif - #ifdef NETDATA_INTERNAL_CHECKS DICTIONARY *dict; #endif @@ -72,15 +57,7 @@ struct dictionary { NAME_VALUE *first_item; // the double linked list base pointers NAME_VALUE *last_item; -#ifdef DICTIONARY_WITH_AVL - avl_tree_type values_index; - NAME_VALUE *hash_base; - void *(*get_thread_static_name_value)(const char *name); -#endif - -#ifdef DICTIONARY_WITH_JUDYHS Pvoid_t JudyHSArray; // the hash table -#endif netdata_rwlock_t rwlock; // the r/w lock when DICTIONARY_FLAG_SINGLE_THREADED is not set @@ -315,9 +292,9 @@ static inline size_t dictionary_lock_free(DICTIONARY *dict) { } static void dictionary_lock(DICTIONARY *dict, char rw) { - if(rw == 'u' || rw == 'U') return; + if(rw == DICTIONARY_LOCK_NONE || rw == 'U') return; - if(rw == 'r' || rw == 'R') { + if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') { // read lock __atomic_add_fetch(&dict->readers, 1, __ATOMIC_RELAXED); } @@ -329,7 +306,7 @@ static void dictionary_lock(DICTIONARY *dict, char rw) { if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) return; - if(rw == 'r' || rw == 'R') { + if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') { // read lock netdata_rwlock_rdlock(&dict->rwlock); @@ -347,9 +324,9 @@ static void dictionary_lock(DICTIONARY *dict, char rw) { } static void dictionary_unlock(DICTIONARY *dict, char rw) { - if(rw == 'u' || rw == 'U') return; + if(rw == DICTIONARY_LOCK_NONE || rw == 'U') return; - if(rw == 'r' || rw == 'R') { + if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') { // read unlock __atomic_sub_fetch(&dict->readers, 1, __ATOMIC_RELAXED); } @@ -480,68 +457,6 @@ static uint32_t reference_counter_release(DICTIONARY *dict, NAME_VALUE *nv, bool // ---------------------------------------------------------------------------- // hash table -#ifdef DICTIONARY_WITH_AVL -static inline const char *namevalue_get_name(NAME_VALUE *nv); - -static int name_value_compare(void* a, void* b) { - return strcmp(namevalue_get_name((NAME_VALUE *)a), namevalue_get_name((NAME_VALUE *)b)); -} - -static void *get_thread_static_name_value(const char *name) { - static __thread NAME_VALUE tmp = { 0 }; - tmp.flags = NAME_VALUE_FLAG_NONE; - tmp.caller_name = (char *)name; - return &tmp; -} - -static void hashtable_init_unsafe(DICTIONARY *dict) { - avl_init(&dict->values_index, name_value_compare); - dict->get_thread_static_name_value = get_thread_static_name_value; -} - -static size_t hashtable_destroy_unsafe(DICTIONARY *dict) { - (void)dict; - return 0; -} - -static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *nv) { - (void)name; - (void)name_len; - - if(unlikely(avl_remove(&(dict->values_index), (avl_t *)(nv)) != (avl_t *)nv)) - return 0; - - return 1; -} - -static inline NAME_VALUE *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { - (void)name_len; - - void *tmp = dict->get_thread_static_name_value(name); - return (NAME_VALUE *)avl_search(&(dict->values_index), (avl_t *)tmp); -} - -static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { - // AVL needs a NAME_VALUE to insert into the dictionary but we don't have it yet. - // So, the only thing we can do, is return an existing one if it is already there. - // Returning NULL will make the caller thing we added it, will allocate one - // and will call hashtable_inserted_name_value_unsafe(), at which we will do - // the actual indexing. - - dict->hash_base = hashtable_get_unsafe(dict, name, name_len); - return &dict->hash_base; -} - -static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, void *nv) { - // we have our new NAME_VALUE object. - // Let's index it. - - if(unlikely(avl_insert(&((dict)->values_index), (avl_t *)(nv)) != (avl_t *)nv)) - error("dictionary: INTERNAL ERROR: duplicate insertion to dictionary."); -} -#endif - -#ifdef DICTIONARY_WITH_JUDYHS static void hashtable_init_unsafe(DICTIONARY *dict) { dict->JudyHSArray = NULL; } @@ -562,7 +477,7 @@ static size_t hashtable_destroy_unsafe(DICTIONARY *dict) { return (size_t)ret; } -static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { +static inline void **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: inserting item from the index without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); JError_t J_Error; @@ -579,7 +494,7 @@ static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char // put anything needed at the value of the index. // The pointer to pointer we return has to be used before // any other operation that may change the index (insert/delete). - return (NAME_VALUE **)Rc; + return Rc; } static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *nv) { @@ -632,8 +547,6 @@ static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, void * ; } -#endif // DICTIONARY_WITH_JUDYHS - // ---------------------------------------------------------------------------- // linked list management @@ -979,7 +892,7 @@ static NAME_VALUE *dictionary_set_name_value_unsafe(DICTIONARY *dict, const char // But the caller has the option to do this on his/her own. // So, let's do the fastest here and let the caller decide the flow of calls. - NAME_VALUE *nv, **pnv = hashtable_insert_unsafe(dict, name, name_len); + NAME_VALUE *nv, **pnv = (NAME_VALUE **)hashtable_insert_unsafe(dict, name, name_len); if(likely(*pnv == 0)) { // a new item added to the index nv = *pnv = namevalue_create_unsafe(dict, name, name_len, value, value_len); @@ -1280,6 +1193,9 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) { dfe->value = NULL; } + if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) + dictionary_unlock(dfe->dict, dfe->rw); + return dfe->value; } @@ -1294,6 +1210,9 @@ void *dictionary_foreach_next(DICTFE *dfe) { return NULL; } + if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) + dictionary_lock(dfe->dict, dfe->rw); + // the item we just did NAME_VALUE *nv = (NAME_VALUE *)dfe->last_item; @@ -1320,6 +1239,9 @@ void *dictionary_foreach_next(DICTFE *dfe) { dfe->value = NULL; } + if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) + dictionary_unlock(dfe->dict, dfe->rw); + return dfe->value; } @@ -1338,7 +1260,9 @@ usec_t dictionary_foreach_done(DICTFE *dfe) { if(likely(nv)) reference_counter_release(dfe->dict, nv, false); - dictionary_unlock(dfe->dict, dfe->rw); + if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT)) + dictionary_unlock(dfe->dict, dfe->rw); + dfe->dict = NULL; dfe->last_item = NULL; dfe->name = NULL; @@ -1383,8 +1307,14 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const c // while we are using it reference_counter_acquire(dict, nv); + if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) + dictionary_unlock(dict, rw); + int r = callback(namevalue_get_name(nv), nv->value, data); + if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) + dictionary_lock(dict, rw); + // since we have a reference counter, this item cannot be deleted // until we release the reference counter, so the pointers are there nv_next = nv->next; @@ -1449,7 +1379,15 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)( nv = array[i]; if(likely(!(nv->flags & NAME_VALUE_FLAG_DELETED))) { reference_counter_acquire(dict, nv); + + if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) + dictionary_unlock(dict, rw); + int r = callback(namevalue_get_name(nv), nv->value, data); + + if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) + dictionary_lock(dict, rw); + reference_counter_release(dict, nv, false); if (r < 0) { ret = r; @@ -1469,132 +1407,293 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)( // ---------------------------------------------------------------------------- // STRING implementation - dedup all STRINGs -typedef struct string_entry { -#ifdef DICTIONARY_WITH_AVL - avl_t avl_node; -#endif - uint32_t length; // the string length with the terminating '\0' - uint32_t refcount; // how many times this string is used - const char str[]; // the string itself -} STRING_ENTRY; +struct netdata_string { + uint32_t length; // the string length including the terminating '\0' -#ifdef DICTIONARY_WITH_AVL -static int string_entry_compare(void* a, void* b) { - return strcmp(((STRING_ENTRY *)a)->str, ((STRING_ENTRY *)b)->str); -} + int32_t refcount; // how many times this string is used + // We use a signed number to be able to detect duplicate frees of a string. + // If at any point this goes below zero, we have a duplicate free. -static void *get_thread_static_string_entry(const char *name) { - static __thread size_t _length = 0; - static __thread STRING_ENTRY *_tmp = NULL; - - size_t size = sizeof(STRING_ENTRY) + strlen(name) + 1; - if(likely(_tmp && _length < size)) { - freez(_tmp); - _tmp = NULL; - _length = 0; - } - - if(unlikely(!_tmp)) { - _tmp = callocz(1, size); - _length = size; - } - - strcpy((char *)&_tmp->str[0], name); - return _tmp; -} -#endif - -DICTIONARY string_dictionary = { -#ifdef DICTIONARY_WITH_AVL - .values_index = { - .root = NULL, - .compar = string_entry_compare - }, - .get_thread_static_name_value = get_thread_static_string_entry, -#endif - - .flags = DICTIONARY_FLAG_EXCLUSIVE_ACCESS, - .rwlock = NETDATA_RWLOCK_INITIALIZER + const char str[]; // the string itself, is appended to this structure }; -static netdata_mutex_t string_mutex = NETDATA_MUTEX_INITIALIZER; +static struct string_hashtable { + Pvoid_t JudyHSArray; // the Judy array - hashtable + netdata_rwlock_t rwlock; // the R/W lock to protect the Judy array + + long int entries; // the number of entries in the index + long int active_references; // the number of active references alive + long int memory; // the memory used, without the JudyHS index + + size_t inserts; // the number of successful inserts to the index + size_t deletes; // the number of successful deleted from the index + size_t searches; // the number of successful searches in the index + size_t duplications; // when a string is referenced + size_t releases; // when a string is unreferenced + +#ifdef NETDATA_INTERNAL_CHECKS + // internal statistics + size_t found_deleted_on_search; + size_t found_available_on_search; + size_t found_deleted_on_insert; + size_t found_available_on_insert; + size_t spins; +#endif + +} string_base = { + .JudyHSArray = NULL, + .rwlock = NETDATA_RWLOCK_INITIALIZER, +}; + +#ifdef NETDATA_INTERNAL_CHECKS +#define string_internal_stats_add(var, val) __atomic_add_fetch(&string_base.var, val, __ATOMIC_RELAXED) +#else +#define string_internal_stats_add(var, val) do {;} while(0) +#endif + +#define string_stats_atomic_increment(var) __atomic_add_fetch(&string_base.var, 1, __ATOMIC_RELAXED) +#define string_stats_atomic_decrement(var) __atomic_sub_fetch(&string_base.var, 1, __ATOMIC_RELAXED) + +void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) { + *inserts = string_base.inserts; + *deletes = string_base.deletes; + *searches = string_base.searches; + *entries = (size_t)string_base.entries; + *references = (size_t)string_base.active_references; + *memory = (size_t)string_base.memory; + *duplications = string_base.duplications; + *releases = string_base.releases; +} + +#define string_entry_acquire(se) __atomic_add_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); +#define string_entry_release(se) __atomic_sub_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); + +static inline bool string_entry_check_and_acquire(STRING *se) { + int32_t expected, desired, count = 0; + do { + count++; + + expected = se->refcount; + + if(expected <= 0) { + // We cannot use this. + // The reference counter reached value zero, + // so another thread is deleting this. + string_internal_stats_add(spins, count - 1); + return false; + } + + desired = expected + 1; + } + while(!__atomic_compare_exchange_n(&se->refcount, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)); + + string_internal_stats_add(spins, count - 1); + + // statistics + // string_base.active_references is altered at the in string_strdupz() and string_freez() + string_stats_atomic_increment(duplications); + + return true; +} STRING *string_dup(STRING *string) { if(unlikely(!string)) return NULL; - STRING_ENTRY *se = (STRING_ENTRY *)string; - netdata_mutex_lock(&string_mutex); - se->refcount++; - netdata_mutex_unlock(&string_mutex); +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) <= 0)) + fatal("STRING: tried to %s() a string that is freed (it has %d references).", __FUNCTION__, string->refcount); +#endif + + string_entry_acquire(string); + + // statistics + string_stats_atomic_increment(active_references); + string_stats_atomic_increment(duplications); + return string; } +// Search the index and return an ACQUIRED string entry, or NULL +static inline STRING *string_index_search(const char *str, size_t length) { + if(unlikely(!string_base.JudyHSArray)) + return NULL; + + STRING *string; + + // Find the string in the index + // With a read-lock so that multiple readers can use the index concurrently. + + netdata_rwlock_rdlock(&string_base.rwlock); + + Pvoid_t *Rc; + Rc = JudyHSGet(string_base.JudyHSArray, (void *)str, length); + if(likely(Rc)) { + // found in the hash table + string = *Rc; + + if(string_entry_check_and_acquire(string)) { + // we can use this entry + string_internal_stats_add(found_available_on_search, 1); + } + else { + // this entry is about to be deleted by another thread + // do not touch it, let it go... + string = NULL; + string_internal_stats_add(found_deleted_on_search, 1); + } + } + else { + // not found in the hash table + string = NULL; + } + + string_stats_atomic_increment(searches); + netdata_rwlock_unlock(&string_base.rwlock); + + return string; +} + +// Insert a string to the index and return an ACQUIRED string entry, +// or NULL if the call needs to be retried (a deleted entry with the same key is still in the index) +// The returned entry is ACQUIRED and it can either be: +// 1. a new item inserted, or +// 2. an item found in the index that is not currently deleted +static inline STRING *string_index_insert(const char *str, size_t length) { + STRING *string; + + netdata_rwlock_wrlock(&string_base.rwlock); + + STRING **ptr; + { + JError_t J_Error; + Pvoid_t *Rc = JudyHSIns(&string_base.JudyHSArray, (void *)str, length, &J_Error); + if (unlikely(Rc == PJERR)) { + fatal( + "STRING: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d", + str, + JU_ERRNO(&J_Error), + JU_ERRID(&J_Error)); + } + ptr = (STRING **)Rc; + } + + if (likely(*ptr == 0)) { + // a new item added to the index + size_t mem_size = sizeof(STRING) + length; + string = mallocz(mem_size); + strcpy((char *)string->str, str); + string->length = length; + string->refcount = 1; + *ptr = string; + string_base.inserts++; + string_base.entries++; + string_base.memory += (long)mem_size; + } + else { + // the item is already in the index + string = *ptr; + + if(string_entry_check_and_acquire(string)) { + // we can use this entry + string_internal_stats_add(found_available_on_insert, 1); + } + else { + // this entry is about to be deleted by another thread + // do not touch it, let it go... + string = NULL; + string_internal_stats_add(found_deleted_on_insert, 1); + } + + string_stats_atomic_increment(searches); + } + + netdata_rwlock_unlock(&string_base.rwlock); + return string; +} + +// delete an entry from the index +static inline void string_index_delete(STRING *string) { + netdata_rwlock_wrlock(&string_base.rwlock); + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) != 0)) + fatal("STRING: tried to delete a string at %s() that is already freed (it has %d references).", __FUNCTION__, string->refcount); +#endif + + bool deleted = false; + + if (likely(string_base.JudyHSArray)) { + JError_t J_Error; + int ret = JudyHSDel(&string_base.JudyHSArray, (void *)string->str, string->length, &J_Error); + if (unlikely(ret == JERR)) { + error( + "STRING: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d", + string->str, + JU_ERRNO(&J_Error), + JU_ERRID(&J_Error)); + } else + deleted = true; + } + + if (unlikely(!deleted)) + error("STRING: tried to delete '%s' that is not in the index. Ignoring it.", string->str); + else { + size_t mem_size = sizeof(STRING) + string->length; + string_base.deletes++; + string_base.entries--; + string_base.memory -= (long)mem_size; + freez(string); + } + + netdata_rwlock_unlock(&string_base.rwlock); +} + STRING *string_strdupz(const char *str) { if(unlikely(!str || !*str)) return NULL; - netdata_mutex_lock(&string_mutex); - size_t length = strlen(str) + 1; - STRING_ENTRY *se; - STRING_ENTRY **ptr = (STRING_ENTRY **)hashtable_insert_unsafe(&string_dictionary, str, length); - if(unlikely(*ptr == 0)) { - // a new item added to the index - size_t mem_size = sizeof(STRING_ENTRY) + length; - se = mallocz(mem_size); - strcpy((char *)se->str, str); - se->length = length; - se->refcount = 1; - *ptr = se; - hashtable_inserted_name_value_unsafe(&string_dictionary, se); - string_dictionary.version++; - string_dictionary.inserts++; - string_dictionary.entries++; - string_dictionary.memory += (long)mem_size; - } - else { - // the item is already in the index - se = *ptr; - se->refcount++; - string_dictionary.searches++; + STRING *string = string_index_search(str, length); + + while(!string) { + // The search above did not find anything, + // We loop here, because during insert we may find an entry that is being deleted by another thread. + // So, we have to let it go and retry to insert it again. + + string = string_index_insert(str, length); } - netdata_mutex_unlock(&string_mutex); - return (STRING *)se; + // statistics + string_stats_atomic_increment(active_references); + + return string; } void string_freez(STRING *string) { if(unlikely(!string)) return; - netdata_mutex_lock(&string_mutex); - STRING_ENTRY *se = (STRING_ENTRY *)string; + int32_t refcount = string_entry_release(string); - if(se->refcount == 0) - fatal("STRING: tried to free string that has zero references."); +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(refcount < 0)) + fatal("STRING: tried to %s() a string that is already freed (it has %d references).", __FUNCTION__, string->refcount); +#endif - se->refcount--; - if(unlikely(se->refcount == 0)) { - if(hashtable_delete_unsafe(&string_dictionary, se->str, se->length, se) == 0) - error("STRING: INTERNAL ERROR: tried to delete '%s' that is not in the index", se->str); + if(unlikely(refcount == 0)) + string_index_delete(string); - size_t mem_size = sizeof(STRING_ENTRY) + se->length; - freez(se); - string_dictionary.version++; - string_dictionary.deletes++; - string_dictionary.entries--; - string_dictionary.memory -= (long)mem_size; - } - - netdata_mutex_unlock(&string_mutex); + // statistics + string_stats_atomic_decrement(active_references); + string_stats_atomic_increment(releases); } -size_t string_length(STRING *string) { +size_t string_strlen(STRING *string) { if(unlikely(!string)) return 0; - return ((STRING_ENTRY *)string)->length - 1; + return string->length - 1; } const char *string2str(STRING *string) { if(unlikely(!string)) return ""; - return ((STRING_ENTRY *)string)->str; + return string->str; } STRING *string_2way_merge(STRING *a, STRING *b) { @@ -1610,9 +1709,9 @@ STRING *string_2way_merge(STRING *a, STRING *b) { if(unlikely(!a)) return string_dup(X); if(unlikely(!b)) return string_dup(X); - size_t alen = string_length(a); - size_t blen = string_length(b); - size_t length = alen + blen + string_length(X) + 1; + size_t alen = string_strlen(a); + size_t blen = string_strlen(b); + size_t length = alen + blen + string_strlen(X) + 1; char buf1[length + 1], buf2[length + 1], *dst1; const char *s1, *s2; @@ -1642,6 +1741,52 @@ STRING *string_2way_merge(STRING *a, STRING *b) { return string_strdupz(buf1); } +// ---------------------------------------------------------------------------- +// THREAD_CACHE + +static __thread Pvoid_t thread_cache_judy_array = NULL; + +void *thread_cache_entry_get_or_set(void *key, + ssize_t key_length, + void *value, + void *(*transform_the_value_before_insert)(void *key, size_t key_length, void *value) + ) { + if(unlikely(!key || !key_length)) return NULL; + + if(key_length == -1) + key_length = (ssize_t)strlen((char *)key) + 1; + + JError_t J_Error; + Pvoid_t *Rc = JudyHSIns(&thread_cache_judy_array, key, key_length, &J_Error); + if (unlikely(Rc == PJERR)) { + fatal("THREAD_CACHE: Cannot insert entry to JudyHS, JU_ERRNO_* == %u, ID == %d", + JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); + } + + if(*Rc == 0) { + // new item added + + *Rc = (transform_the_value_before_insert) ? transform_the_value_before_insert(key, key_length, value) : value; + } + + return *Rc; +} + +void thread_cache_destroy(void) { + if(unlikely(!thread_cache_judy_array)) return; + + JError_t J_Error; + Word_t ret = JudyHSFreeArray(&thread_cache_judy_array, &J_Error); + if(unlikely(ret == (Word_t) JERR)) { + error("THREAD_CACHE: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d", + JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); + } + + internal_error(true, "THREAD_CACHE: hash table freed %lu bytes", ret); + + thread_cache_judy_array = NULL; +} + // ---------------------------------------------------------------------------- // unit test @@ -2153,6 +2298,27 @@ static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, co return errors; } +static int string_threads_join = 0; +static void *string_thread(void *arg __maybe_unused) { + int dups = 1; //(gettid() % 10); + for(; 1 ;) { + if(string_threads_join) + break; + + STRING *s = string_strdupz("string thread checking 1234567890"); + + for(int i = 0; i < dups ; i++) + string_dup(s); + + for(int i = 0; i < dups ; i++) + string_freez(s); + + string_freez(s); + } + + return arg; +} + int dictionary_unittest(size_t entries) { if(entries < 10) entries = 10; @@ -2291,7 +2457,7 @@ int dictionary_unittest(size_t entries) { // check string { - long string_entries_starting = dictionary_stats_entries(&string_dictionary); + long int string_entries_starting = string_base.entries; fprintf(stderr, "\nChecking strings...\n"); @@ -2312,8 +2478,7 @@ int dictionary_unittest(size_t entries) { else fprintf(stderr, "OK: cloning string are deduplicated\n"); - STRING_ENTRY *se = (STRING_ENTRY *)s1; - if(se->refcount != 3) { + if(s1->refcount != 3) { errors++; fprintf(stderr, "ERROR: string refcount is not 3\n"); } @@ -2355,9 +2520,9 @@ int dictionary_unittest(size_t entries) { freez(strings); - if(dictionary_stats_entries(&string_dictionary) != string_entries_starting + 2) { + if(string_base.entries != string_entries_starting + 2) { errors++; - fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, dictionary_stats_entries(&string_dictionary)); + fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, string_base.entries); } else fprintf(stderr, "OK: strings dictionary has 2 items\n"); @@ -2405,6 +2570,65 @@ int dictionary_unittest(size_t entries) { dictionary_unittest_free_char_pp(names, entries); dictionary_unittest_free_char_pp(values, entries); + { +#ifdef NETDATA_INTERNAL_CHECKS + size_t ofound_deleted_on_search = string_base.found_deleted_on_search, + ofound_available_on_search = string_base.found_available_on_search, + ofound_deleted_on_insert = string_base.found_deleted_on_insert, + ofound_available_on_insert = string_base.found_available_on_insert, + ospins = string_base.spins; +#endif + + size_t oinserts, odeletes, osearches, oentries, oreferences, omemory, oduplications, oreleases; + string_statistics(&oinserts, &odeletes, &osearches, &oentries, &oreferences, &omemory, &oduplications, &oreleases); + + time_t seconds_to_run = 5; + int threads_to_create = 2; + fprintf( + stderr, + "Checking string concurrency with %d threads for %ld seconds...\n", + threads_to_create, + seconds_to_run); + // check string concurrency + netdata_thread_t threads[threads_to_create]; + string_threads_join = 0; + for (int i = 0; i < threads_to_create; i++) { + char buf[100 + 1]; + snprintf(buf, 100, "string%d", i); + netdata_thread_create( + &threads[i], buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, string_thread, NULL); + } + sleep_usec(seconds_to_run * USEC_PER_SEC); + + string_threads_join = 1; + for (int i = 0; i < threads_to_create; i++) { + void *retval; + netdata_thread_join(threads[i], &retval); + } + + size_t inserts, deletes, searches, sentries, references, memory, duplications, releases; + string_statistics(&inserts, &deletes, &searches, &sentries, &references, &memory, &duplications, &releases); + + fprintf(stderr, "inserts %zu, deletes %zu, searches %zu, entries %zu, references %zu, memory %zu, duplications %zu, releases %zu\n", + inserts - oinserts, deletes - odeletes, searches - osearches, sentries - oentries, references - oreferences, memory - omemory, duplications - oduplications, releases - oreleases); + +#ifdef NETDATA_INTERNAL_CHECKS + size_t found_deleted_on_search = string_base.found_deleted_on_search, + found_available_on_search = string_base.found_available_on_search, + found_deleted_on_insert = string_base.found_deleted_on_insert, + found_available_on_insert = string_base.found_available_on_insert, + spins = string_base.spins; + + fprintf(stderr, "on insert: %zu ok + %zu deleted\non search: %zu ok + %zu deleted\nspins: %zu\n", + found_available_on_insert - ofound_available_on_insert, + found_deleted_on_insert - ofound_deleted_on_insert, + found_available_on_search - ofound_available_on_search, + found_deleted_on_search - ofound_deleted_on_search, + spins - ospins + ); +#endif + } + fprintf(stderr, "\n%zu errors found\n", errors); return errors ? 1 : 0; } diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h index fdb2088c0b..9c2ea72e41 100644 --- a/libnetdata/dictionary/dictionary.h +++ b/libnetdata/dictionary/dictionary.h @@ -47,9 +47,9 @@ typedef enum dictionary_flags { DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end) // to change the value of the following, you also need to change the corresponding #defines in dictionary.c - DICTIONARY_FLAG_RESERVED1 = (1 << 29), // reserved for DICTIONARY_FLAG_EXCLUSIVE_ACCESS - DICTIONARY_FLAG_RESERVED2 = (1 << 30), // reserved for DICTIONARY_FLAG_DESTROYED - DICTIONARY_FLAG_RESERVED3 = (1 << 31), // reserved for DICTIONARY_FLAG_DEFER_ALL_DELETIONS + DICTIONARY_FLAG_RESERVED1 = (1 << 28), // reserved for DICTIONARY_FLAG_EXCLUSIVE_ACCESS + DICTIONARY_FLAG_RESERVED2 = (1 << 29), // reserved for DICTIONARY_FLAG_DESTROYED + DICTIONARY_FLAG_RESERVED3 = (1 << 30), // reserved for DICTIONARY_FLAG_DEFER_ALL_DELETIONS } DICTIONARY_FLAGS; // Create a dictionary @@ -179,9 +179,10 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)( #define DICTFE_CONST const #endif -#define DICTIONARY_LOCK_READ 'r' -#define DICTIONARY_LOCK_WRITE 'w' -#define DICTIONARY_LOCK_NONE 'u' +#define DICTIONARY_LOCK_READ 'r' +#define DICTIONARY_LOCK_WRITE 'w' +#define DICTIONARY_LOCK_REENTRANT 'z' +#define DICTIONARY_LOCK_NONE 'u' typedef DICTFE_CONST struct dictionary_foreach { DICTFE_CONST char *name; // the dictionary name of the last item used @@ -235,7 +236,7 @@ typedef struct netdata_string STRING; extern STRING *string_strdupz(const char *str); extern STRING *string_dup(STRING *string); extern void string_freez(STRING *string); -extern size_t string_length(STRING *string); +extern size_t string_strlen(STRING *string); extern const char *string2str(STRING *string) NEVERNULL; // keep common prefix/suffix and replace everything else with [x] @@ -243,10 +244,20 @@ extern STRING *string_2way_merge(STRING *a, STRING *b); static inline int string_cmp(STRING *s1, STRING *s2) { // STRINGs are deduplicated, so the same strings have the same pointer - if(unlikely(s1 == s2)) return 0; - - // they differ, do the typical comparison - return strcmp(string2str(s1), string2str(s2)); + // when they differ, we do the typical strcmp() comparison + return (s1 == s2)?0:strcmp(string2str(s1), string2str(s2)); } +extern void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases); + +// ---------------------------------------------------------------------------- +// THREAD CACHE + +extern void *thread_cache_entry_get_or_set(void *key, + ssize_t key_length, + void *value, + void *(*transform_the_value_before_insert)(void *key, size_t key_length, void *value)); + +extern void thread_cache_destroy(void); + #endif /* NETDATA_DICTIONARY_H */ diff --git a/libnetdata/eval/eval.c b/libnetdata/eval/eval.c index e86cbd587e..0e429a08cb 100644 --- a/libnetdata/eval/eval.c +++ b/libnetdata/eval/eval.c @@ -62,24 +62,36 @@ static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n); // evaluation of expressions static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) { - static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0; + static STRING + *this_string = NULL, + *now_string = NULL, + *after_string = NULL, + *before_string = NULL, + *status_string = NULL, + *removed_string = NULL, + *uninitialized_string = NULL, + *undefined_string = NULL, + *clear_string = NULL, + *warning_string = NULL, + *critical_string = NULL; + NETDATA_DOUBLE n; - if(unlikely(this_hash == 0)) { - this_hash = simple_hash("this"); - now_hash = simple_hash("now"); - after_hash = simple_hash("after"); - before_hash = simple_hash("before"); - status_hash = simple_hash("status"); - removed_hash = simple_hash("REMOVED"); - uninitialized_hash = simple_hash("UNINITIALIZED"); - undefined_hash = simple_hash("UNDEFINED"); - clear_hash = simple_hash("CLEAR"); - warning_hash = simple_hash("WARNING"); - critical_hash = simple_hash("CRITICAL"); + if(unlikely(this_string == NULL)) { + this_string = string_strdupz("this"); + now_string = string_strdupz("now"); + after_string = string_strdupz("after"); + before_string = string_strdupz("before"); + status_string = string_strdupz("status"); + removed_string = string_strdupz("REMOVED"); + uninitialized_string = string_strdupz("UNINITIALIZED"); + undefined_string = string_strdupz("UNDEFINED"); + clear_string = string_strdupz("CLEAR"); + warning_string = string_strdupz("WARNING"); + critical_string = string_strdupz("CRITICAL"); } - if(unlikely(v->hash == this_hash && !strcmp(v->name, "this"))) { + if(unlikely(v->name == this_string)) { n = (exp->myself)?*exp->myself:NAN; buffer_strcat(exp->error_msg, "[ $this = "); print_parsed_as_constant(exp->error_msg, n); @@ -87,7 +99,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == after_hash && !strcmp(v->name, "after"))) { + if(unlikely(v->name == after_string)) { n = (exp->after && *exp->after)?*exp->after:NAN; buffer_strcat(exp->error_msg, "[ $after = "); print_parsed_as_constant(exp->error_msg, n); @@ -95,7 +107,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == before_hash && !strcmp(v->name, "before"))) { + if(unlikely(v->name == before_string)) { n = (exp->before && *exp->before)?*exp->before:NAN; buffer_strcat(exp->error_msg, "[ $before = "); print_parsed_as_constant(exp->error_msg, n); @@ -103,15 +115,15 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == now_hash && !strcmp(v->name, "now"))) { - n = now_realtime_sec(); + if(unlikely(v->name == now_string)) { + n = (NETDATA_DOUBLE)now_realtime_sec(); buffer_strcat(exp->error_msg, "[ $now = "); print_parsed_as_constant(exp->error_msg, n); buffer_strcat(exp->error_msg, " ] "); return n; } - if(unlikely(v->hash == status_hash && !strcmp(v->name, "status"))) { + if(unlikely(v->name == status_string)) { n = (exp->status)?*exp->status:RRDCALC_STATUS_UNINITIALIZED; buffer_strcat(exp->error_msg, "[ $status = "); print_parsed_as_constant(exp->error_msg, n); @@ -119,7 +131,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == removed_hash && !strcmp(v->name, "REMOVED"))) { + if(unlikely(v->name == removed_string)) { n = RRDCALC_STATUS_REMOVED; buffer_strcat(exp->error_msg, "[ $REMOVED = "); print_parsed_as_constant(exp->error_msg, n); @@ -127,7 +139,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == uninitialized_hash && !strcmp(v->name, "UNINITIALIZED"))) { + if(unlikely(v->name == uninitialized_string)) { n = RRDCALC_STATUS_UNINITIALIZED; buffer_strcat(exp->error_msg, "[ $UNINITIALIZED = "); print_parsed_as_constant(exp->error_msg, n); @@ -135,7 +147,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == undefined_hash && !strcmp(v->name, "UNDEFINED"))) { + if(unlikely(v->name == undefined_string)) { n = RRDCALC_STATUS_UNDEFINED; buffer_strcat(exp->error_msg, "[ $UNDEFINED = "); print_parsed_as_constant(exp->error_msg, n); @@ -143,7 +155,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == clear_hash && !strcmp(v->name, "CLEAR"))) { + if(unlikely(v->name == clear_string)) { n = RRDCALC_STATUS_CLEAR; buffer_strcat(exp->error_msg, "[ $CLEAR = "); print_parsed_as_constant(exp->error_msg, n); @@ -151,7 +163,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == warning_hash && !strcmp(v->name, "WARNING"))) { + if(unlikely(v->name == warning_string)) { n = RRDCALC_STATUS_WARNING; buffer_strcat(exp->error_msg, "[ $WARNING = "); print_parsed_as_constant(exp->error_msg, n); @@ -159,7 +171,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(unlikely(v->hash == critical_hash && !strcmp(v->name, "CRITICAL"))) { + if(unlikely(v->name == critical_string)) { n = RRDCALC_STATUS_CRITICAL; buffer_strcat(exp->error_msg, "[ $CRITICAL = "); print_parsed_as_constant(exp->error_msg, n); @@ -167,15 +179,15 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE * return n; } - if(exp->rrdcalc && health_variable_lookup(v->name, v->hash, exp->rrdcalc, &n)) { - buffer_sprintf(exp->error_msg, "[ ${%s} = ", v->name); + if(exp->rrdcalc && health_variable_lookup(v->name, exp->rrdcalc, &n)) { + buffer_sprintf(exp->error_msg, "[ ${%s} = ", string2str(v->name)); print_parsed_as_constant(exp->error_msg, n); buffer_strcat(exp->error_msg, " ] "); return n; } *error = EVAL_ERROR_UNKNOWN_VARIABLE; - buffer_sprintf(exp->error_msg, "[ undefined variable '%s' ] ", v->name); + buffer_sprintf(exp->error_msg, "[ undefined variable '%s' ] ", string2str(v->name)); return NAN; } @@ -357,7 +369,7 @@ static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *error) { (void)error; - buffer_sprintf(out, "${%s}", v->name); + buffer_sprintf(out, "${%s}", string2str(v->name)); } static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n) { @@ -859,12 +871,11 @@ static inline void eval_node_set_value_to_variable(EVAL_NODE *op, int pos, const op->ops[pos].type = EVAL_VALUE_VARIABLE; op->ops[pos].variable = callocz(1, sizeof(EVAL_VARIABLE)); - op->ops[pos].variable->name = strdupz(variable); - op->ops[pos].variable->hash = simple_hash(op->ops[pos].variable->name); + op->ops[pos].variable->name = string_strdupz(variable); } static inline void eval_variable_free(EVAL_VARIABLE *v) { - freez(v->name); + string_freez(v->name); freez(v); } diff --git a/libnetdata/eval/eval.h b/libnetdata/eval/eval.h index 086d171aad..b3c13d193d 100644 --- a/libnetdata/eval/eval.h +++ b/libnetdata/eval/eval.h @@ -18,8 +18,7 @@ typedef enum rrdcalc_status { } RRDCALC_STATUS; typedef struct eval_variable { - char *name; - uint32_t hash; + STRING *name; struct eval_variable *next; } EVAL_VARIABLE; @@ -83,6 +82,6 @@ extern const char *expression_strerror(int error); // 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg) extern int expression_evaluate(EVAL_EXPRESSION *expression); -extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result); +extern int health_variable_lookup(STRING *variable, struct rrdcalc *rc, NETDATA_DOUBLE *result); #endif //NETDATA_EVAL_H diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index 8cc6cce9f7..b1dbad3c86 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -214,6 +214,76 @@ extern "C" { #define GUID_LEN 36 +// --------------------------------------------------------------------------------------------- +// double linked list management + +#define DOUBLE_LINKED_LIST_PREPEND_UNSAFE(head, item, prev, next) \ + do { \ + (item)->next = (head); \ + \ + if(likely(head)) { \ + (item)->prev = (head)->prev; \ + (head)->prev = (item); \ + } \ + else \ + (item)->prev = (item); \ + \ + (head) = (item); \ + } while (0) + +#define DOUBLE_LINKED_LIST_APPEND_UNSAFE(head, item, prev, next) \ + do { \ + if(likely(head)) { \ + (item)->prev = (head)->prev; \ + (head)->prev->next = (item); \ + (head)->prev = (item); \ + (item)->next = NULL; \ + } \ + else { \ + (head) = (item); \ + (head)->prev = (head); \ + (head)->next = NULL; \ + } \ + \ + } while (0) + +#define DOUBLE_LINKED_LIST_REMOVE_UNSAFE(head, item, prev, next) \ + do { \ + fatal_assert((head) != NULL); \ + fatal_assert((item)->prev != NULL); \ + \ + if((item)->prev == (item)) { \ + /* it is the only item in the list */ \ + (head) = NULL; \ + } \ + else if((item) == (head)) { \ + /* it is the first item */ \ + (item)->next->prev = (item)->prev; \ + (head) = (item)->next; \ + } \ + else { \ + (item)->prev->next = (item)->next; \ + if ((item)->next) { \ + (item)->next->prev = (item)->prev; \ + } \ + else { \ + (head)->prev = (item)->prev; \ + } \ + } \ + \ + (item)->next = NULL; \ + (item)->prev = NULL; \ + } while (0) + +#define DOUBLE_LINKED_LIST_FOREACH_FORWARD(head, var, prev, next) \ + for ((var) = (head); (var) ; (var) = (var)->next) + +#define DOUBLE_LINKED_LIST_FOREACH_BACKWARD(head, var, prev, next) \ + for ((var) = (head)?(head)->prev:NULL; (var) && (var) != (head)->prev ; (var) = (var)->prev) + +// --------------------------------------------------------------------------------------------- + + extern void netdata_fix_chart_id(char *s); extern void netdata_fix_chart_name(char *s); diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c index d6793b69bf..4035031861 100644 --- a/libnetdata/log/log.c +++ b/libnetdata/log/log.c @@ -747,7 +747,7 @@ void debug_int( const char *file, const char *function, const unsigned long line // ---------------------------------------------------------------------------- // info log -void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) +void info_int( const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { va_list args; @@ -807,7 +807,7 @@ static const char *strerror_result_string(const char *a, const char *b) { (void) #error "cannot detect the format of function strerror_r()" #endif -void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { +void error_int( const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { // save a copy of errno - just in case this function generates a new error int __errno = errno; diff --git a/libnetdata/required_dummies.h b/libnetdata/required_dummies.h index 6d51bfedd4..3a10f1feca 100644 --- a/libnetdata/required_dummies.h +++ b/libnetdata/required_dummies.h @@ -22,15 +22,16 @@ void signals_block(void){}; void signals_unblock(void){}; void signals_reset(void){}; +#ifndef UNIT_TESTING // callback required by eval() -int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result) +int health_variable_lookup(STRING *variable, struct rrdcalc *rc, NETDATA_DOUBLE *result) { (void)variable; - (void)hash; (void)rc; (void)result; return 0; }; +#endif // required by get_system_cpus() char *netdata_configured_host_prefix = ""; diff --git a/libnetdata/simple_pattern/simple_pattern.c b/libnetdata/simple_pattern/simple_pattern.c index 70b06a22bc..81c2ed0b8c 100644 --- a/libnetdata/simple_pattern/simple_pattern.c +++ b/libnetdata/simple_pattern/simple_pattern.c @@ -333,9 +333,7 @@ extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p) } char *simple_pattern_trim_around_equal(char *src) { - char *store = mallocz(strlen(src) +1); - if(!store) - return NULL; + char *store = mallocz(strlen(src) + 1); char *dst = store; while (*src) { diff --git a/libnetdata/threads/threads.c b/libnetdata/threads/threads.c index 12007afff5..58ba765a4d 100644 --- a/libnetdata/threads/threads.c +++ b/libnetdata/threads/threads.c @@ -106,6 +106,8 @@ static void thread_cleanup(void *ptr) { if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP)) info("thread with task id %d finished", gettid()); + thread_cache_destroy(); + freez((void *)netdata_thread->tag); netdata_thread->tag = NULL; diff --git a/libnetdata/worker_utilization/worker_utilization.c b/libnetdata/worker_utilization/worker_utilization.c index bd3ad60e09..7df050f386 100644 --- a/libnetdata/worker_utilization/worker_utilization.c +++ b/libnetdata/worker_utilization/worker_utilization.c @@ -4,15 +4,20 @@ #define WORKER_BUSY 'B' struct worker_job_type { - char name[WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH + 1]; + STRING *name; + STRING *units; // statistics controlled variables size_t statistics_last_jobs_started; usec_t statistics_last_busy_time; + NETDATA_DOUBLE statistics_last_custom_value; // worker controlled variables volatile size_t worker_jobs_started; volatile usec_t worker_busy_time; + + WORKER_METRIC_TYPE type; + NETDATA_DOUBLE custom_value; }; struct worker { @@ -36,6 +41,7 @@ struct worker { struct worker_job_type per_job_type[WORKER_UTILIZATION_MAX_JOB_TYPES]; struct worker *next; + struct worker *prev; }; static netdata_mutex_t base_lock = NETDATA_MUTEX_INITIALIZER; @@ -57,40 +63,44 @@ void worker_register(const char *workname) { worker->last_action = WORKER_IDLE; netdata_mutex_lock(&base_lock); - worker->next = base; - base = worker; + DOUBLE_LINKED_LIST_PREPEND_UNSAFE(base, worker, prev, next); netdata_mutex_unlock(&base_lock); } -void worker_register_job_name(size_t job_id, const char *name) { +void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type) { if(unlikely(!worker)) return; if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES)) { error("WORKER_UTILIZATION: job_id %zu is too big. Max is %zu", job_id, (size_t)(WORKER_UTILIZATION_MAX_JOB_TYPES - 1)); return; } - if (*worker->per_job_type[job_id].name) { - error("WORKER_UTILIZATION: duplicate job registration: worker '%s' job id %zu is '%s', ignoring '%s'", worker->workname, job_id, worker->per_job_type[job_id].name, name); + if(worker->per_job_type[job_id].name) { + if(strcmp(string2str(worker->per_job_type[job_id].name), name) != 0 || worker->per_job_type[job_id].type != type || strcmp(string2str(worker->per_job_type[job_id].units), units) != 0) + error("WORKER_UTILIZATION: duplicate job registration: worker '%s' job id %zu is '%s', ignoring the later '%s'", worker->workname, job_id, string2str(worker->per_job_type[job_id].name), name); return; } - strncpy(worker->per_job_type[job_id].name, name, WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH); + worker->per_job_type[job_id].name = string_strdupz(name); + worker->per_job_type[job_id].units = string_strdupz(units); + worker->per_job_type[job_id].type = type; +} + +void worker_register_job_name(size_t job_id, const char *name) { + worker_register_job_custom_metric(job_id, name, "", WORKER_METRIC_IDLE_BUSY); } void worker_unregister(void) { if(unlikely(!worker)) return; netdata_mutex_lock(&base_lock); - if(base == worker) - base = worker->next; - else { - struct worker *p; - for(p = base; p && p->next && p->next != worker ;p = p->next); - if(p && p->next == worker) - p->next = worker->next; - } + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(base, worker, prev, next); netdata_mutex_unlock(&base_lock); + for(int i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) { + string_freez(worker->per_job_type[i].name); + string_freez(worker->per_job_type[i].units); + } + freez((void *)worker->tag); freez((void *)worker->workname); freez(worker); @@ -112,16 +122,14 @@ static inline void worker_is_idle_with_time(usec_t now) { } void worker_is_idle(void) { - if(unlikely(!worker)) return; - if(unlikely(worker->last_action != WORKER_BUSY)) return; + if(unlikely(!worker || worker->last_action != WORKER_BUSY)) return; worker_is_idle_with_time(now_realtime_usec()); } void worker_is_busy(size_t job_id) { - if(unlikely(!worker)) return; - if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES)) - job_id = 0; + if(unlikely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES)) + return; usec_t now = now_realtime_usec(); @@ -138,35 +146,97 @@ void worker_is_busy(size_t job_id) { worker->last_action = WORKER_BUSY; } +void worker_set_metric(size_t job_id, NETDATA_DOUBLE value) { + if(unlikely(!worker)) return; + if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES)) + return; + + if(worker->per_job_type[job_id].type == WORKER_METRIC_INCREMENTAL) + worker->per_job_type[job_id].custom_value += value; + else + worker->per_job_type[job_id].custom_value = value; +} // statistics interface -void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pid, const char *thread_tag, size_t utilization_usec, size_t duration_usec, size_t jobs_started, size_t is_running, const char **job_types_names, size_t *job_types_jobs_started, usec_t *job_types_busy_time), void *data) { +void workers_foreach(const char *workname, void (*callback)( + void *data + , pid_t pid + , const char *thread_tag + , size_t utilization_usec + , size_t duration_usec + , size_t jobs_started, size_t is_running + , STRING **job_types_names + , STRING **job_types_units + , WORKER_METRIC_TYPE *job_metric_types + , size_t *job_types_jobs_started + , usec_t *job_types_busy_time + , NETDATA_DOUBLE *job_custom_values + ) + , void *data) { netdata_mutex_lock(&base_lock); uint32_t hash = simple_hash(workname); usec_t busy_time, delta; size_t i, jobs_started, jobs_running; struct worker *p; - for(p = base; p ; p = p->next) { - if(hash != p->workname_hash || strcmp(workname, p->workname)) continue; + DOUBLE_LINKED_LIST_FOREACH_FORWARD(base, p, prev, next) { + if(hash != p->workname_hash || strcmp(workname, p->workname) != 0) continue; usec_t now = now_realtime_usec(); // find per job type statistics - const char *per_job_type_name[WORKER_UTILIZATION_MAX_JOB_TYPES]; + STRING *per_job_type_name[WORKER_UTILIZATION_MAX_JOB_TYPES]; + STRING *per_job_type_units[WORKER_UTILIZATION_MAX_JOB_TYPES]; + WORKER_METRIC_TYPE per_job_metric_type[WORKER_UTILIZATION_MAX_JOB_TYPES]; size_t per_job_type_jobs_started[WORKER_UTILIZATION_MAX_JOB_TYPES]; usec_t per_job_type_busy_time[WORKER_UTILIZATION_MAX_JOB_TYPES]; + NETDATA_DOUBLE per_job_custom_values[WORKER_UTILIZATION_MAX_JOB_TYPES]; + for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) { per_job_type_name[i] = p->per_job_type[i].name; + per_job_type_units[i] = p->per_job_type[i].units; + per_job_metric_type[i] = p->per_job_type[i].type; - size_t tmp_jobs_started = p->per_job_type[i].worker_jobs_started; - per_job_type_jobs_started[i] = tmp_jobs_started - p->per_job_type[i].statistics_last_jobs_started; - p->per_job_type[i].statistics_last_jobs_started = tmp_jobs_started; + switch(p->per_job_type[i].type) { + default: + case WORKER_METRIC_EMPTY: + per_job_type_jobs_started[i] = 0; + per_job_type_busy_time[i] = 0; + per_job_custom_values[i] = NAN; + break; - usec_t tmp_busy_time = p->per_job_type[i].worker_busy_time; - per_job_type_busy_time[i] = tmp_busy_time - p->per_job_type[i].statistics_last_busy_time; - p->per_job_type[i].statistics_last_busy_time = tmp_busy_time; + case WORKER_METRIC_IDLE_BUSY: { + size_t tmp_jobs_started = p->per_job_type[i].worker_jobs_started; + per_job_type_jobs_started[i] = tmp_jobs_started - p->per_job_type[i].statistics_last_jobs_started; + p->per_job_type[i].statistics_last_jobs_started = tmp_jobs_started; + + usec_t tmp_busy_time = p->per_job_type[i].worker_busy_time; + per_job_type_busy_time[i] = tmp_busy_time - p->per_job_type[i].statistics_last_busy_time; + p->per_job_type[i].statistics_last_busy_time = tmp_busy_time; + + per_job_custom_values[i] = NAN; + break; + } + + case WORKER_METRIC_ABSOLUTE: + per_job_type_jobs_started[i] = 0; + per_job_type_busy_time[i] = 0; + + per_job_custom_values[i] = p->per_job_type[i].custom_value; + break; + + case WORKER_METRIC_INCREMENTAL: { + per_job_type_jobs_started[i] = 0; + per_job_type_busy_time[i] = 0; + + NETDATA_DOUBLE tmp_custom_value = p->per_job_type[i].custom_value; + per_job_custom_values[i] = tmp_custom_value - p->per_job_type[i].statistics_last_custom_value; + p->per_job_type[i].statistics_last_custom_value = tmp_custom_value; + + break; + } + } } // get a copy of the worker variables @@ -203,7 +273,20 @@ void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pi jobs_running = 1; } - callback(data, p->pid, p->tag, busy_time, delta, jobs_started, jobs_running, per_job_type_name, per_job_type_jobs_started, per_job_type_busy_time); + callback(data + , p->pid + , p->tag + , busy_time + , delta + , jobs_started + , jobs_running + , per_job_type_name + , per_job_type_units + , per_job_metric_type + , per_job_type_jobs_started + , per_job_type_busy_time + , per_job_custom_values + ); } netdata_mutex_unlock(&base_lock); diff --git a/libnetdata/worker_utilization/worker_utilization.h b/libnetdata/worker_utilization/worker_utilization.h index 8f16fe0549..b64d76a9ed 100644 --- a/libnetdata/worker_utilization/worker_utilization.h +++ b/libnetdata/worker_utilization/worker_utilization.h @@ -6,17 +6,40 @@ // workers interfaces #define WORKER_UTILIZATION_MAX_JOB_TYPES 50 -#define WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH 25 + +typedef enum { + WORKER_METRIC_EMPTY = 0, + WORKER_METRIC_IDLE_BUSY = 1, + WORKER_METRIC_ABSOLUTE = 2, + WORKER_METRIC_INCREMENTAL = 3, +} WORKER_METRIC_TYPE; extern void worker_register(const char *workname); extern void worker_register_job_name(size_t job_id, const char *name); +extern void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type); extern void worker_unregister(void); extern void worker_is_idle(void); extern void worker_is_busy(size_t job_id); +extern void worker_set_metric(size_t job_id, NETDATA_DOUBLE value); // statistics interface -extern void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pid, const char *thread_tag, size_t utilization_usec, size_t duration_usec, size_t jobs_started, size_t is_running, const char **job_types_names, size_t *job_types_jobs_started, usec_t *job_types_busy_time), void *data); +extern void workers_foreach(const char *workname, void (*callback)( + void *data + , pid_t pid + , const char *thread_tag + , size_t utilization_usec + , size_t duration_usec + , size_t jobs_started + , size_t is_running + , STRING **job_types_names + , STRING **job_types_units + , WORKER_METRIC_TYPE *job_metric_types + , size_t *job_types_jobs_started + , usec_t *job_types_busy_time + , NETDATA_DOUBLE *job_custom_values + ) + , void *data); #endif // WORKER_UTILIZATION_H diff --git a/ml/Dimension.h b/ml/Dimension.h index 4fbc09b981..1cc053df3f 100644 --- a/ml/Dimension.h +++ b/ml/Dimension.h @@ -26,7 +26,7 @@ public: RRDSET *RS = RD->rrdset; std::stringstream SS; - SS << RS->context << "|" << RS->id << "|" << RD->name; + SS << rrdset_context(RS) << "|" << rrdset_id(RS) << "|" << rrddim_name(RD); return SS.str(); } diff --git a/ml/Host.cc b/ml/Host.cc index f8cba9d64e..a6f9b330a7 100644 --- a/ml/Host.cc +++ b/ml/Host.cc @@ -23,7 +23,7 @@ static void updateDimensionsChart(RRDHOST *RH, std::stringstream IdSS, NameSS; IdSS << "dimensions_on_" << localhost->machine_guid; - NameSS << "dimensions_on_" << localhost->hostname; + NameSS << "dimensions_on_" << rrdhost_hostname(localhost); RS = rrdset_create( RH, @@ -69,7 +69,7 @@ static void updateRateChart(RRDHOST *RH, collected_number AnomalyRate) { std::stringstream IdSS, NameSS; IdSS << "anomaly_rate_on_" << localhost->machine_guid; - NameSS << "anomaly_rate_on_" << localhost->hostname; + NameSS << "anomaly_rate_on_" << rrdhost_hostname(localhost); RS = rrdset_create( RH, @@ -106,7 +106,7 @@ static void updateWindowLengthChart(RRDHOST *RH, collected_number WindowLength) std::stringstream IdSS, NameSS; IdSS << "detector_window_on_" << localhost->machine_guid; - NameSS << "detector_window_on_" << localhost->hostname; + NameSS << "detector_window_on_" << rrdhost_hostname(localhost); RS = rrdset_create( RH, @@ -147,7 +147,7 @@ static void updateEventsChart(RRDHOST *RH, std::stringstream IdSS, NameSS; IdSS << "detector_events_on_" << localhost->machine_guid; - NameSS << "detector_events_on_" << localhost->hostname; + NameSS << "detector_events_on_" << rrdhost_hostname(localhost); RS = rrdset_create( RH, @@ -193,7 +193,7 @@ static void updateDetectionChart(RRDHOST *RH) { std::stringstream IdSS, NameSS; IdSS << "prediction_stats_" << RH->machine_guid; - NameSS << "prediction_stats_for_" << RH->hostname; + NameSS << "prediction_stats_for_" << rrdhost_hostname(RH); RS = rrdset_create_localhost( "netdata", // type @@ -233,7 +233,7 @@ static void updateTrainingChart(RRDHOST *RH, struct rusage *TRU) std::stringstream IdSS, NameSS; IdSS << "training_stats_" << RH->machine_guid; - NameSS << "training_stats_for_" << RH->hostname; + NameSS << "training_stats_for_" << rrdhost_hostname(RH); RS = rrdset_create_localhost( "netdata", // type diff --git a/ml/Host.h b/ml/Host.h index 2715008f09..5fd8318fd1 100644 --- a/ml/Host.h +++ b/ml/Host.h @@ -31,9 +31,7 @@ public: RRDSET_TYPE_LINE ); - AnomalyRateRS->flags = static_cast( - static_cast(AnomalyRateRS->flags) | RRDSET_FLAG_HIDDEN - ); + rrdset_flag_set(AnomalyRateRS, RRDSET_FLAG_HIDDEN); } RRDHOST *getRH() { return RH; } diff --git a/ml/ml.cc b/ml/ml.cc index 7275d88b87..8c7f56dd73 100644 --- a/ml/ml.cc +++ b/ml/ml.cc @@ -16,7 +16,7 @@ bool ml_enabled(RRDHOST *RH) { if (!Cfg.EnableAnomalyDetection) return false; - if (simple_pattern_matches(Cfg.SP_HostsToSkip, RH->hostname)) + if (simple_pattern_matches(Cfg.SP_HostsToSkip, rrdhost_hostname(RH))) return false; return true; @@ -76,7 +76,7 @@ void ml_new_dimension(RRDDIM *RD) { if (static_cast(RD->update_every) != H->updateEvery()) return; - if (simple_pattern_matches(Cfg.SP_ChartsToSkip, RS->name)) + if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RS))) return; Dimension *D = new Dimension(RD); diff --git a/parser/parser.c b/parser/parser.c index c37d1e2c43..45026e5143 100644 --- a/parser/parser.c +++ b/parser/parser.c @@ -38,6 +38,7 @@ PARSER *parser_init(RRDHOST *host, void *user, void *input, PARSER_INPUT_TYPE fl parser->input = input; parser->flags = flags; parser->host = host; + parser->worker_job_next_id = WORKER_PARSER_FIRST_JOB; #ifdef ENABLE_HTTPS parser->bytesleft = 0; @@ -133,7 +134,7 @@ int parser_add_keyword(PARSER *parser, char *keyword, keyword_function func) tmp_keyword = callocz(1, sizeof(*tmp_keyword)); - tmp_keyword->worker_job_id = parser->worker_job_ids++; + tmp_keyword->worker_job_id = parser->worker_job_next_id++; tmp_keyword->keyword = strdupz(keyword); tmp_keyword->keyword_hash = keyword_hash; tmp_keyword->func[tmp_keyword->func_no++] = (void *) func; @@ -268,7 +269,7 @@ inline int parser_action(PARSER *parser, char *input) uint32_t command_hash = simple_hash(command); - size_t worker_job_id = 0; + size_t worker_job_id = WORKER_UTILIZATION_MAX_JOB_TYPES + 1; // set an invalid value by default while(tmp_keyword) { if (command_hash == tmp_keyword->keyword_hash && (!strcmp(command, tmp_keyword->keyword))) { diff --git a/parser/parser.h b/parser/parser.h index 1887318fc7..c64475f9c5 100644 --- a/parser/parser.h +++ b/parser/parser.h @@ -7,6 +7,7 @@ #define PARSER_MAX_CALLBACKS 20 #define PARSER_MAX_RECOVER_KEYWORDS 128 +#define WORKER_PARSER_FIRST_JOB 1 // PARSER return codes typedef enum parser_rc { @@ -68,7 +69,7 @@ typedef struct parser_data { } PARSER_DATA; typedef struct parser { - size_t worker_job_ids; + size_t worker_job_next_id; uint8_t version; // Parser version RRDHOST *host; void *input; // Input source e.g. stream diff --git a/registry/registry.c b/registry/registry.c index a4f9c6de32..afacdfebe1 100644 --- a/registry/registry.c +++ b/registry/registry.c @@ -61,7 +61,7 @@ static inline void registry_json_header(RRDHOST *host, struct web_client *w, con buffer_flush(w->response.data); w->response.data->contenttype = CT_APPLICATION_JSON; buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"", - action, status, host->registry_hostname, host->machine_guid); + action, status, rrdhost_registry_hostname(host), host->machine_guid); } static inline void registry_json_footer(struct web_client *w) { diff --git a/streaming/compression.c b/streaming/compression.c index d6178d6c34..302b0b1809 100644 --- a/streaming/compression.c +++ b/streaming/compression.c @@ -5,9 +5,6 @@ #define STREAM_COMPRESSION_MSG "STREAM_COMPRESSION" -#define LZ4_MAX_MSG_SIZE 0x4000 -#define LZ4_STREAM_BUFFER_SIZE (0x10000 + LZ4_MAX_MSG_SIZE) - #define SIGNATURE ((uint32_t)('z' | 0x80) | (0x80 << 8) | (0x80 << 16) | ('\n' << 24)) #define SIGNATURE_MASK ((uint32_t)0xff | (0x80 << 8) | (0x80 << 16) | (0xff << 24)) #define SIGNATURE_SIZE 4 @@ -18,8 +15,9 @@ */ struct compressor_data { LZ4_stream_t *stream; - char *stream_buffer; - size_t stream_buffer_pos; + char *input_ring_buffer; + size_t input_ring_buffer_size; + size_t input_ring_buffer_pos; }; @@ -33,7 +31,7 @@ static void lz4_compressor_reset(struct compressor_state *state) LZ4_resetStream_fast(state->data->stream); info("%s: Compressor Reset", STREAM_COMPRESSION_MSG); } - state->data->stream_buffer_pos = 0; + state->data->input_ring_buffer_pos = 0; } } @@ -47,10 +45,10 @@ static void lz4_compressor_destroy(struct compressor_state **state) if (s->data) { if (s->data->stream) LZ4_freeStream(s->data->stream); - freez(s->data->stream_buffer); + freez(s->data->input_ring_buffer); freez(s->data); } - freez(s->buffer); + freez(s->compression_result_buffer); freez(s); *state = NULL; debug(D_STREAM, "%s: Compressor Destroyed.", STREAM_COMPRESSION_MSG); @@ -65,37 +63,53 @@ static void lz4_compressor_destroy(struct compressor_state **state) */ static size_t lz4_compressor_compress(struct compressor_state *state, const char *data, size_t size, char **out) { - if (!state || !size || !out) + if(unlikely(!state || !size || !out)) return 0; - if (size > LZ4_MAX_MSG_SIZE) { + + if(unlikely(size > LZ4_MAX_MSG_SIZE)) { error("%s: Compression Failed - Message size %lu above compression buffer limit: %d", STREAM_COMPRESSION_MSG, size, LZ4_MAX_MSG_SIZE); return 0; } + size_t max_dst_size = LZ4_COMPRESSBOUND(size); size_t data_size = max_dst_size + SIGNATURE_SIZE; - if (!state->buffer) { - state->buffer = mallocz(data_size); - state->buffer_size = data_size; - } else if (state->buffer_size < data_size) { - state->buffer = reallocz(state->buffer, data_size); - state->buffer_size = data_size; + if (!state->compression_result_buffer) { + state->compression_result_buffer = mallocz(data_size); + state->compression_result_buffer_size = data_size; + } + else if(unlikely(state->compression_result_buffer_size < data_size)) { + state->compression_result_buffer = reallocz(state->compression_result_buffer, data_size); + state->compression_result_buffer_size = data_size; } - memcpy(state->data->stream_buffer + state->data->stream_buffer_pos, data, size); - long int compressed_data_size = LZ4_compress_fast_continue(state->data->stream, - state->data->stream_buffer + state->data->stream_buffer_pos, - state->buffer + SIGNATURE_SIZE, size, max_dst_size, 1); + // the ring buffer always has space for LZ4_MAX_MSG_SIZE + memcpy(state->data->input_ring_buffer + state->data->input_ring_buffer_pos, data, size); + + // this call needs the last 64K of our previous data + // they are available in the ring buffer + long int compressed_data_size = LZ4_compress_fast_continue( + state->data->stream, + state->data->input_ring_buffer + state->data->input_ring_buffer_pos, + state->compression_result_buffer + SIGNATURE_SIZE, + size, + max_dst_size, + 1); + if (compressed_data_size < 0) { error("Data compression error: %ld", compressed_data_size); return 0; } - state->data->stream_buffer_pos += size; - if (state->data->stream_buffer_pos >= LZ4_STREAM_BUFFER_SIZE - LZ4_MAX_MSG_SIZE) - state->data->stream_buffer_pos = 0; + + // update the next writing position of the ring buffer + state->data->input_ring_buffer_pos += size; + if(unlikely(state->data->input_ring_buffer_pos >= state->data->input_ring_buffer_size - LZ4_MAX_MSG_SIZE)) + state->data->input_ring_buffer_pos = 0; + + // update the signature header uint32_t len = ((compressed_data_size & 0x7f) | 0x80 | (((compressed_data_size & (0x7f << 7)) << 1) | 0x8000)) << 8; - *(uint32_t *)state->buffer = len | SIGNATURE; - *out = state->buffer; + *(uint32_t *)state->compression_result_buffer = len | SIGNATURE; + *out = state->compression_result_buffer; debug(D_STREAM, "%s: Compressed data header: %ld", STREAM_COMPRESSION_MSG, compressed_data_size); return compressed_data_size + SIGNATURE_SIZE; } @@ -114,8 +128,9 @@ struct compressor_state *create_compressor() state->data = callocz(1, sizeof(struct compressor_data)); state->data->stream = LZ4_createStream(); - state->data->stream_buffer = callocz(1, LZ4_DECODER_RING_BUFFER_SIZE(LZ4_MAX_MSG_SIZE)); - state->buffer_size = LZ4_STREAM_BUFFER_SIZE; + state->data->input_ring_buffer_size = LZ4_DECODER_RING_BUFFER_SIZE(LZ4_MAX_MSG_SIZE * 2); + state->data->input_ring_buffer = callocz(1, state->data->input_ring_buffer_size); + state->compression_result_buffer_size = 0; state->reset(state); debug(D_STREAM, "%s: Initialize streaming compression!", STREAM_COMPRESSION_MSG); return state; @@ -281,6 +296,8 @@ static size_t lz4_decompressor_decompress(struct decompressor_state *state) size_t avg_saving = saving_percent(state->total_compressed, state->total_uncompressed); size_t avg_size = state->total_uncompressed / state->packet_count; + (void)saving; + if (old_avg_saving != avg_saving || old_avg_size != avg_size){ debug(D_STREAM, "%s: Saving: %lu%% (avg. %lu%%), avg.size: %lu", STREAM_COMPRESSION_MSG, saving, avg_saving, avg_size); } diff --git a/streaming/receiver.c b/streaming/receiver.c index 0890ebbcd0..a2852981a2 100644 --- a/streaming/receiver.c +++ b/streaming/receiver.c @@ -1,6 +1,13 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "rrdpush.h" +#include "parser/parser.h" + +#define WORKER_RECEIVER_JOB_BYTES_READ (WORKER_PARSER_FIRST_JOB - 1) + +#if WORKER_PARSER_FIRST_JOB < 1 +#error The define WORKER_PARSER_FIRST_JOB needs to be at least 1 +#endif extern struct config stream_config; @@ -66,7 +73,7 @@ PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host; struct plugind *cd = ((PARSER_USER_OBJECT *)user)->cd; if (cd->version < VERSION_GAP_FILLING ) { - error("STREAM %s from %s: Child negotiated version %u but sent TIMESTAMP!", host->hostname, cd->cmd, + error("STREAM %s from %s: Child negotiated version %u but sent TIMESTAMP!", rrdhost_hostname(host), cd->cmd, cd->version); return PARSER_RC_OK; // Ignore error and continue stream } @@ -78,7 +85,7 @@ PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins info( "STREAM %s from %s: Initial connection (no gap to check), " "remote=%"PRId64" local=%"PRId64" slew=%"PRId64"", - host->hostname, + rrdhost_hostname(host), cd->cmd, (int64_t)remote_time, (int64_t)now, @@ -88,7 +95,7 @@ PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins info( "STREAM %s from %s: Checking for gaps... " "remote=%"PRId64" local=%"PRId64"..%"PRId64" slew=%"PRId64" %"PRId64"-sec gap", - host->hostname, + rrdhost_hostname(host), cd->cmd, (int64_t)remote_time, (int64_t)prev, @@ -177,6 +184,7 @@ static int receiver_read(struct receiver_state *r, FILE *fp) { int ret = SSL_read(r->ssl.conn, r->read_buffer + r->read_len, desired); if (ret > 0 ) { r->read_len += ret; + worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret); return 0; } // Don't treat SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE differently on blocking socket @@ -192,6 +200,7 @@ static int receiver_read(struct receiver_state *r, FILE *fp) { if (!fgets(r->read_buffer, sizeof(r->read_buffer), fp)) return 1; r->read_len = strlen(r->read_buffer); + worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, r->read_len); return 0; } #else @@ -279,22 +288,24 @@ static int read_stream(struct receiver_state *r, FILE *fp, char* buffer, size_t */ static int receiver_read(struct receiver_state *r, FILE *fp) { // check any decompressed data present - if (r->decompressor && - r->decompressor->decompressed_bytes_in_buffer(r->decompressor)) { + if (r->decompressor && r->decompressor->decompressed_bytes_in_buffer(r->decompressor)) { size_t available = sizeof(r->read_buffer) - r->read_len; if (available) { - size_t len = r->decompressor->get(r->decompressor, - r->read_buffer + r->read_len, available); + size_t len = r->decompressor->get(r->decompressor, r->read_buffer + r->read_len, available); if (!len) return 1; + r->read_len += len; } return 0; } + int ret = 0; if (read_stream(r, fp, r->read_buffer + r->read_len, sizeof(r->read_buffer) - r->read_len - 1, &ret)) return 1; + worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret); + if (!is_compressed_data(r->read_buffer, ret)) { r->read_len += ret; return 0; @@ -303,8 +314,7 @@ static int receiver_read(struct receiver_state *r, FILE *fp) { if (unlikely(!r->decompressor)) r->decompressor = create_decompressor(); - size_t bytes_to_read = r->decompressor->start(r->decompressor, - r->read_buffer, ret); + size_t bytes_to_read = r->decompressor->start(r->decompressor, r->read_buffer, ret); // Read the entire block of compressed data because // we're unable to decompress incomplete block @@ -312,18 +322,23 @@ static int receiver_read(struct receiver_state *r, FILE *fp) { do { if (read_stream(r, fp, compressed, bytes_to_read, &ret)) return 1; + + worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret); + // Send input data to decompressor if (ret) r->decompressor->put(r->decompressor, compressed, ret); + bytes_to_read -= ret; } while (bytes_to_read > 0); + // Decompress size_t bytes_to_parse = r->decompressor->decompress(r->decompressor); if (!bytes_to_parse) return 1; + // Fill read buffer with decompressed data - r->read_len = r->decompressor->get(r->decompressor, - r->read_buffer, sizeof(r->read_buffer)); + r->read_len = r->decompressor->get(r->decompressor, r->read_buffer, sizeof(r->read_buffer)); return 0; } @@ -486,8 +501,8 @@ static int rrdpush_receive(struct receiver_state *rpt) #else if(send_timeout(rpt->fd, initial_response, strlen(initial_response), 0, 60) != strlen(initial_response)) { #endif - log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "FAILED - CANNOT REPLY"); - error("STREAM %s [receive from [%s]:%s]: cannot send command.", rpt->host->hostname, rpt->client_ip, rpt->client_port); + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - CANNOT REPLY"); + error("STREAM %s [receive from [%s]:%s]: cannot send command.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port); close(rpt->fd); return 0; } @@ -575,14 +590,14 @@ static int rrdpush_receive(struct receiver_state *rpt) , rpt->hostname , rpt->client_ip , rpt->client_port - , rpt->host->hostname + , rrdhost_hostname(rpt->host) , rpt->host->machine_guid , rpt->host->rrd_update_every , rpt->host->rrd_history_entries , rrd_memory_mode_name(rpt->host->rrd_memory_mode) , (health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto") , ssl ? " SSL," : "" - , rpt->host->tags?rpt->host->tags:"" + , rrdhost_tags(rpt->host) ); #endif // NETDATA_INTERNAL_CHECKS @@ -605,7 +620,7 @@ static int rrdpush_receive(struct receiver_state *rpt) snprintfz(cd.fullfilename, FILENAME_MAX, "%s:%s", rpt->client_ip, rpt->client_port); snprintfz(cd.cmd, PLUGINSD_CMD_MAX, "%s:%s", rpt->client_ip, rpt->client_port); - info("STREAM %s [receive from [%s]:%s]: initializing communication...", rpt->host->hostname, rpt->client_ip, rpt->client_port); + info("STREAM %s [receive from [%s]:%s]: initializing communication...", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port); char initial_response[HTTP_HEADER_SIZE]; if (rpt->stream_version > 1) { if(rpt->stream_version >= STREAM_VERSION_COMPRESSION){ @@ -618,13 +633,13 @@ static int rrdpush_receive(struct receiver_state *rpt) } #endif } - info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->stream_version); + info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->stream_version); sprintf(initial_response, "%s%u", START_STREAMING_PROMPT_VN, rpt->stream_version); } else if (rpt->stream_version == 1) { - info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->stream_version); + info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->stream_version); sprintf(initial_response, "%s", START_STREAMING_PROMPT_V2); } else { - info("STREAM %s [receive from [%s]:%s]: Netdata is using first stream protocol.", rpt->host->hostname, rpt->client_ip, rpt->client_port); + info("STREAM %s [receive from [%s]:%s]: Netdata is using first stream protocol.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port); sprintf(initial_response, "%s", START_STREAMING_PROMPT); } debug(D_STREAM, "Initial response to %s: %s", rpt->client_ip, initial_response); @@ -635,27 +650,27 @@ static int rrdpush_receive(struct receiver_state *rpt) #else if(send_timeout(rpt->fd, initial_response, strlen(initial_response), 0, 60) != strlen(initial_response)) { #endif - log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "FAILED - CANNOT REPLY"); - error("STREAM %s [receive from [%s]:%s]: cannot send ready command.", rpt->host->hostname, rpt->client_ip, rpt->client_port); + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - CANNOT REPLY"); + error("STREAM %s [receive from [%s]:%s]: cannot send ready command.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port); close(rpt->fd); return 0; } // remove the non-blocking flag from the socket if(sock_delnonblock(rpt->fd) < 0) - error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->fd); + error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd); struct timeval timeout; timeout.tv_sec = 120; timeout.tv_usec = 0; if (unlikely(setsockopt(rpt->fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout) != 0)) - error("STREAM %s [receive from [%s]:%s]: cannot set timeout for socket %d", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->fd); + error("STREAM %s [receive from [%s]:%s]: cannot set timeout for socket %d", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd); // convert the socket to a FILE * FILE *fp = fdopen(rpt->fd, "r"); if(!fp) { - log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "FAILED - SOCKET ERROR"); - error("STREAM %s [receive from [%s]:%s]: failed to get a FILE for FD %d.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->fd); + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - SOCKET ERROR"); + error("STREAM %s [receive from [%s]:%s]: failed to get a FILE for FD %d.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd); close(rpt->fd); return 0; } @@ -686,7 +701,7 @@ static int rrdpush_receive(struct receiver_state *rpt) info( "Postponing health checks for %" PRId64 " seconds, on host '%s', because it was just connected.", (int64_t)alarms_delay, - rpt->host->hostname); + rrdhost_hostname(rpt->host)); } } rpt->host->senders_connect_time = now_realtime_sec(); @@ -695,8 +710,8 @@ static int rrdpush_receive(struct receiver_state *rpt) rrdhost_unlock(rpt->host); // call the plugins.d processor to receive the metrics - info("STREAM %s [receive from [%s]:%s]: receiving metrics...", rpt->host->hostname, rpt->client_ip, rpt->client_port); - log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "CONNECTED"); + info("STREAM %s [receive from [%s]:%s]: receiving metrics...", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port); + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "CONNECTED"); cd.version = rpt->stream_version; @@ -758,6 +773,7 @@ void *rrdpush_receiver_thread(void *ptr) { info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid()); worker_register("STREAMRCV"); + worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_READ, "received bytes", "bytes/s", WORKER_METRIC_INCREMENTAL); rrdpush_receive(rpt); worker_unregister(); diff --git a/streaming/rrdpush.c b/streaming/rrdpush.c index b73f24633c..67fc48aa09 100644 --- a/streaming/rrdpush.c +++ b/streaming/rrdpush.c @@ -140,8 +140,8 @@ static inline int should_send_chart_matching(RRDSET *st) { if(!rrdset_flag_check(st, RRDSET_FLAG_UPSTREAM_SEND|RRDSET_FLAG_UPSTREAM_IGNORE)) { RRDHOST *host = st->rrdhost; - if(simple_pattern_matches(host->rrdpush_send_charts_matching, st->id) || - simple_pattern_matches(host->rrdpush_send_charts_matching, st->name)) { + if(simple_pattern_matches(host->rrdpush_send_charts_matching, rrdset_id(st)) || + simple_pattern_matches(host->rrdpush_send_charts_matching, rrdset_name(st))) { rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE); rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_SEND); } @@ -175,20 +175,17 @@ int configured_as_parent() { // checks if the current chart definition has been sent static inline int need_to_send_chart_definition(RRDSET *st) { - rrdset_check_rdlock(st); - if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_UPSTREAM_EXPOSED)))) return 1; RRDDIM *rd; - rrddim_foreach_read(rd, st) { + dfe_start_read(st->rrddim_root_index, rd) { if(unlikely(!rd->exposed)) { - #ifdef NETDATA_INTERNAL_CHECKS - info("host '%s', chart '%s', dimension '%s' flag 'exposed' triggered chart refresh to upstream", st->rrdhost->hostname, st->id, rd->id); - #endif + internal_error(true, "host '%s', chart '%s', dimension '%s' flag 'exposed' triggered chart refresh to upstream", rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(rd)); return 1; } } + dfe_done(rd); return 0; } @@ -216,9 +213,9 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { // properly set the name for the remote end to parse it char *name = ""; if(likely(st->name)) { - if(unlikely(strcmp(st->id, st->name))) { + if(unlikely(st->id != st->name)) { // they differ - name = strchr(st->name, '.'); + name = strchr(rrdset_name(st), '.'); if(name) name++; else @@ -230,12 +227,12 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { buffer_sprintf( host->sender->build , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s %s\" \"%s\" \"%s\"\n" - , st->id + , rrdset_id(st) , name - , st->title - , st->units - , st->family - , st->context + , rrdset_title(st) + , rrdset_units(st) + , rrdset_family(st) + , rrdset_context(st) , rrdset_type_name(st->chart_type) , st->priority , st->update_every @@ -243,8 +240,8 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":"" , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":"" , rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)?"hidden":"" - , (st->plugin_name)?st->plugin_name:"" - , (st->module_name)?st->module_name:"" + , rrdset_plugin_name(st) + , rrdset_module_name(st) ); // send the chart labels @@ -257,8 +254,8 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { buffer_sprintf( host->sender->build , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n" - , rd->id - , rd->name + , rrddim_id(rd) + , rrddim_name(rd) , rrd_algorithm_name(rd->algorithm) , rd->multiplier , rd->divisor @@ -278,7 +275,7 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { buffer_sprintf( host->sender->build , "VARIABLE CHART %s = " NETDATA_DOUBLE_FORMAT "\n" - , rs->variable + , string2str(rs->variable) , *value ); } @@ -288,40 +285,75 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { } // sends the current chart dimensions -static inline void rrdpush_send_chart_metrics_nolock(RRDSET *st, struct sender_state *s) { +static inline bool rrdpush_send_chart_metrics_nolock(RRDSET *st, struct sender_state *s) { RRDHOST *host = st->rrdhost; - buffer_sprintf(host->sender->build, "BEGIN \"%s\" %llu", st->id, (st->last_collected_time.tv_sec > st->upstream_resync_time)?st->usec_since_last_update:0); + buffer_sprintf(host->sender->build, "BEGIN \"%s\" %llu", rrdset_id(st), (st->last_collected_time.tv_sec > st->upstream_resync_time)?st->usec_since_last_update:0); if (s->version >= VERSION_GAP_FILLING) buffer_sprintf(host->sender->build, " %"PRId64"\n", (int64_t)st->last_collected_time.tv_sec); else buffer_strcat(host->sender->build, "\n"); + size_t count_of_dimensions_written = 0; RRDDIM *rd; rrddim_foreach_read(rd, st) { - if(rd->updated && rd->exposed) - buffer_sprintf(host->sender->build - , "SET \"%s\" = " COLLECTED_NUMBER_FORMAT "\n" - , rd->id - , rd->collected_value - ); + if(rd->updated && rd->exposed) { + buffer_sprintf(host->sender->build, "SET \"%s\" = " COLLECTED_NUMBER_FORMAT "\n", rrddim_id(rd), rd->collected_value); + count_of_dimensions_written++; + } } buffer_strcat(host->sender->build, "END\n"); + + return count_of_dimensions_written != 0; } static void rrdpush_sender_thread_spawn(RRDHOST *host); // Called from the internal collectors to mark a chart obsolete. -void rrdset_push_chart_definition_now(RRDSET *st) { +bool rrdset_push_chart_definition_now(RRDSET *st) { RRDHOST *host = st->rrdhost; if(unlikely(!host->rrdpush_send_enabled || !should_send_chart_matching(st))) - return; + return false; rrdset_rdlock(st); sender_start(host->sender); rrdpush_send_chart_definition_nolock(st); sender_commit(host->sender); rrdset_unlock(st); + + return true; +} + +bool rrdpush_incremental_transmission_of_chart_definitions(RRDHOST *host, DICTFE *dictfe, bool restart, bool stop) { + if(stop || restart) + dictionary_foreach_done(dictfe); + + if(stop) + return false; + + RRDSET *st = NULL; + + if(unlikely(!dictfe->dict)) { + st = dictionary_foreach_start_rw(dictfe, host->rrdset_root_index, DICTIONARY_LOCK_REENTRANT); + } + else + st = dictionary_foreach_next(dictfe); + + do { + while(st && !need_to_send_chart_definition(st)) + st = dictionary_foreach_next(dictfe); + + if(st && rrdset_push_chart_definition_now(st)) + break; + + } while((st = dictionary_foreach_next(dictfe))); + + if (!st) { + dictionary_foreach_done(dictfe); + return false; + } + + return true; } void rrdset_done_push(RRDSET *st) { @@ -334,29 +366,37 @@ void rrdset_done_push(RRDSET *st) { rrdpush_sender_thread_spawn(host); // Handle non-connected case - if(unlikely(!__atomic_load_n(&host->rrdpush_sender_connected, __ATOMIC_SEQ_CST))) { + if(unlikely(!__atomic_load_n(&host->rrdpush_sender_connected, __ATOMIC_SEQ_CST) + || !rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS))) { + if(unlikely(!host->rrdpush_sender_error_shown)) - error("STREAM %s [send]: not ready - discarding collected metrics.", host->hostname); + error("STREAM %s [send]: not ready - collected metrics are not sent to parent.", rrdhost_hostname(host)); host->rrdpush_sender_error_shown = 1; + return; } else if(unlikely(host->rrdpush_sender_error_shown)) { - info("STREAM %s [send]: sending metrics...", host->hostname); + info("STREAM %s [send]: sending metrics to parent...", rrdhost_hostname(host)); host->rrdpush_sender_error_shown = 0; } + if(dictionary_stats_entries(st->rrddim_root_index) == 0) + return; + sender_start(host->sender); if(need_to_send_chart_definition(st)) rrdpush_send_chart_definition_nolock(st); - rrdpush_send_chart_metrics_nolock(st, host->sender); + if(rrdpush_send_chart_metrics_nolock(st, host->sender)) { + // signal the sender there are more data + if (host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1) + error("STREAM %s [send]: cannot write to internal pipe", rrdhost_hostname(host)); - // signal the sender there are more data - if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1) - error("STREAM %s [send]: cannot write to internal pipe", host->hostname); - - sender_commit(host->sender); + sender_commit(host->sender); + } + else + sender_cancel(host->sender); } // labels @@ -376,7 +416,7 @@ void rrdpush_send_labels(RRDHOST *host) { sender_commit(host->sender); if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1) - error("STREAM %s [send]: cannot write to internal pipe", host->hostname); + error("STREAM %s [send]: cannot write to internal pipe", rrdhost_hostname(host)); rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE); } @@ -399,7 +439,7 @@ void rrdpush_claimed_id(RRDHOST *host) // signal the sender there are more data if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1) - error("STREAM %s [send]: cannot write to internal pipe", host->hostname); + error("STREAM %s [send]: cannot write to internal pipe", rrdhost_hostname(host)); } int connect_to_one_of_destinations( @@ -496,7 +536,7 @@ void rrdpush_sender_thread_stop(RRDHOST *host) { netdata_thread_t thr = 0; if(host->rrdpush_sender_spawn) { - info("STREAM %s [send]: signaling sending thread to stop...", host->hostname); + info("STREAM %s [send]: signaling sending thread to stop...", rrdhost_hostname(host)); // signal the thread that we want to join it host->rrdpush_sender_join = 1; @@ -512,10 +552,10 @@ void rrdpush_sender_thread_stop(RRDHOST *host) { netdata_mutex_unlock(&host->sender->mutex); if(thr != 0) { - info("STREAM %s [send]: waiting for the sending thread to stop...", host->hostname); + info("STREAM %s [send]: waiting for the sending thread to stop...", rrdhost_hostname(host)); void *result; netdata_thread_join(thr, &result); - info("STREAM %s [send]: sending thread has exited.", host->hostname); + info("STREAM %s [send]: sending thread has exited.", rrdhost_hostname(host)); } } @@ -533,10 +573,10 @@ static void rrdpush_sender_thread_spawn(RRDHOST *host) { if(!host->rrdpush_sender_spawn) { char tag[NETDATA_THREAD_TAG_MAX + 1]; - snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STREAM_SENDER[%s]", host->hostname); + snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STREAM_SENDER[%s]", rrdhost_hostname(host)); if(netdata_thread_create(&host->rrdpush_sender_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, rrdpush_sender_thread, (void *) host->sender)) - error("STREAM %s [send]: failed to create new thread for client.", host->hostname); + error("STREAM %s [send]: failed to create new thread for client.", rrdhost_hostname(host)); else host->rrdpush_sender_spawn = 1; } @@ -746,7 +786,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) { struct receiver_state *rpt = callocz(1, sizeof(*rpt)); rrd_rdlock(); - RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0); + RRDHOST *host = rrdhost_find_by_guid(machine_guid); if (unlikely(host && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) /* Ignore archived hosts. */ host = NULL; if (host) { @@ -763,7 +803,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) { info( "STREAM %s [receive from [%s]:%s]: multiple connections for same host detected - " "existing connection is dead (%"PRId64" sec), accepting new connection.", - host->hostname, + rrdhost_hostname(host), w->client_ip, w->client_port, (int64_t)age); @@ -772,12 +812,12 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) { netdata_mutex_unlock(&host->receiver_lock); rrdhost_unlock(host); rrd_unlock(); - log_stream_connection(w->client_ip, w->client_port, key, host->machine_guid, host->hostname, + log_stream_connection(w->client_ip, w->client_port, key, host->machine_guid, rrdhost_hostname(host), "REJECTED - ALREADY CONNECTED"); info( "STREAM %s [receive from [%s]:%s]: multiple connections for same host detected - " "existing connection is active (within last %"PRId64" sec), rejecting new connection.", - host->hostname, + rrdhost_hostname(host), w->client_ip, w->client_port, (int64_t)age); diff --git a/streaming/rrdpush.h b/streaming/rrdpush.h index 1eb39cc6cd..68dab2a1a0 100644 --- a/streaming/rrdpush.h +++ b/streaming/rrdpush.h @@ -46,9 +46,10 @@ typedef struct { } stream_encoded_t; #ifdef ENABLE_COMPRESSION +#define LZ4_MAX_MSG_SIZE 0x4000 struct compressor_state { - char *buffer; - size_t buffer_size; + char *compression_result_buffer; + size_t compression_result_buffer_size; struct compressor_data *data; // Compression API specific data void (*reset)(struct compressor_state *state); size_t (*compress)(struct compressor_state *state, const char *data, size_t size, char **buffer); @@ -164,10 +165,12 @@ extern void sender_init(RRDHOST *parent); extern struct rrdpush_destinations *destinations_init(const char *destinations); void sender_start(struct sender_state *s); void sender_commit(struct sender_state *s); +void sender_cancel(struct sender_state *s); extern int rrdpush_init(); extern int configured_as_parent(); extern void rrdset_done_push(RRDSET *st); -extern void rrdset_push_chart_definition_now(RRDSET *st); +extern bool rrdset_push_chart_definition_now(RRDSET *st); +extern bool rrdpush_incremental_transmission_of_chart_definitions(RRDHOST *host, DICTFE *dictfe, bool restart, bool stop); extern void *rrdpush_sender_thread(void *ptr); extern void rrdpush_send_labels(RRDHOST *host); extern void rrdpush_claimed_id(RRDHOST *host); diff --git a/streaming/sender.c b/streaming/sender.c index c4836aeafb..a4d7b78dfd 100644 --- a/streaming/sender.c +++ b/streaming/sender.c @@ -17,9 +17,12 @@ #define WORKER_SENDER_JOB_DISCONNECT_RECEIVE_ERROR 12 #define WORKER_SENDER_JOB_DISCONNECT_SEND_ERROR 13 #define WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION 14 +#define WORKER_SENDER_JOB_BUFFER_RATIO 15 +#define WORKER_SENDER_JOB_BYTES_RECEIVED 16 +#define WORKER_SENDER_JOB_BYTES_SENT 17 -#if WORKER_UTILIZATION_MAX_JOB_TYPES < 15 -#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 15 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 18 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 18 #endif extern struct config stream_config; @@ -33,6 +36,11 @@ void sender_start(struct sender_state *s) { buffer_flush(s->build); } +void sender_cancel(struct sender_state *s) { + buffer_flush(s->build); + netdata_mutex_unlock(&s->mutex); +} + static inline void rrdpush_sender_thread_close_socket(RRDHOST *host); #ifdef ENABLE_COMPRESSION @@ -43,11 +51,11 @@ static inline void rrdpush_sender_thread_close_socket(RRDHOST *host); */ static inline void deactivate_compression(struct sender_state *s) { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION); - error("STREAM_COMPRESSION: Deactivating compression to avoid stream corruption"); + error("STREAM_COMPRESSION: Compression returned error, disabling it."); default_compression_enabled = 0; s->rrdpush_compression = 0; s->version = STREAM_VERSION_CLABELS; - error("STREAM_COMPRESSION %s [send to %s]: Restarting connection without compression", s->host->hostname, s->connected_to); + error("STREAM %s [send to %s]: Restarting connection without compression.", rrdhost_hostname(s->host), s->connected_to); rrdpush_sender_thread_close_socket(s->host); } #endif @@ -59,27 +67,55 @@ void sender_commit(struct sender_state *s) { #ifdef ENABLE_COMPRESSION if (src && src_len) { if (s->compressor && s->rrdpush_compression) { - src_len = s->compressor->compress(s->compressor, src, src_len, &src); - if (!src_len) { - deactivate_compression(s); - buffer_flush(s->build); - netdata_mutex_unlock(&s->mutex); - return; + while(src_len) { + size_t size_to_compress = src_len; + + if(size_to_compress > LZ4_MAX_MSG_SIZE) { + // we need to find the last newline + // so that the decompressor will have a whole line to work with + + const char *t = &src[LZ4_MAX_MSG_SIZE - 1]; + while(t-- > src) + if(*t == '\n') + break; + + if(t == src) + size_to_compress = LZ4_MAX_MSG_SIZE; + else + size_to_compress = t - src + 1; + } + + char *dst; + size_t dst_len = s->compressor->compress(s->compressor, src, size_to_compress, &dst); + if (!dst_len) { + deactivate_compression(s); + buffer_flush(s->build); + netdata_mutex_unlock(&s->mutex); + return; + } + + if(cbuffer_add_unsafe(s->host->sender->buffer, dst, dst_len)) + s->overflow = 1; + + src = src + size_to_compress; + src_len -= size_to_compress; } } - if(cbuffer_add_unsafe(s->host->sender->buffer, src, src_len)) + else if(cbuffer_add_unsafe(s->host->sender->buffer, src, src_len)) s->overflow = 1; } #else if(cbuffer_add_unsafe(s->host->sender->buffer, src, src_len)) s->overflow = 1; #endif + buffer_flush(s->build); netdata_mutex_unlock(&s->mutex); } static inline void rrdpush_sender_thread_close_socket(RRDHOST *host) { + rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS); __atomic_clear(&host->rrdpush_sender_connected, __ATOMIC_SEQ_CST); if(host->rrdpush_sender_socket != -1) { @@ -94,11 +130,11 @@ static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *ho buffer_sprintf( host->sender->build , "VARIABLE HOST %s = " NETDATA_DOUBLE_FORMAT "\n" - , rv->name + , rrdvar_name(rv) , *value ); - debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rv->name, *value); + debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rrdvar_name(rv), *value); } void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) { @@ -110,7 +146,7 @@ void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) { } -static int rrdpush_sender_thread_custom_host_variables_callback(void *rrdvar_ptr, void *host_ptr) { +static int rrdpush_sender_thread_custom_host_variables_callback(const char *name __maybe_unused, void *rrdvar_ptr, void *host_ptr) { RRDVAR *rv = (RRDVAR *)rrdvar_ptr; RRDHOST *host = (RRDHOST *)host_ptr; @@ -127,7 +163,7 @@ static int rrdpush_sender_thread_custom_host_variables_callback(void *rrdvar_ptr static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) { sender_start(host->sender); - int ret = rrdvar_callback_for_all_host_variables(host, rrdpush_sender_thread_custom_host_variables_callback, host); + int ret = rrdvar_walkthrough_read(host->rrdvar_root_index, rrdpush_sender_thread_custom_host_variables_callback, host); (void)ret; sender_commit(host->sender); @@ -162,7 +198,7 @@ static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) { size_t len = cbuffer_next_unsafe(host->sender->buffer, NULL); if (len) - error("STREAM %s [send]: discarding %zu bytes of metrics already in the buffer.", host->hostname, len); + error("STREAM %s [send]: discarding %zu bytes of metrics already in the buffer.", rrdhost_hostname(host), len); cbuffer_remove_unsafe(host->sender->buffer, len); netdata_mutex_unlock(&host->sender->mutex); @@ -259,7 +295,7 @@ static int rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_po rrdpush_sender_thread_close_socket(host); debug(D_STREAM, "STREAM: Attempting to connect..."); - info("STREAM %s [send to %s]: connecting...", host->hostname, host->rrdpush_send_destination); + info("STREAM %s [send to %s]: connecting...", rrdhost_hostname(host), host->rrdpush_send_destination); host->rrdpush_sender_socket = connect_to_one_of_destinations( host->destinations @@ -272,11 +308,11 @@ static int rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_po ); if(unlikely(host->rrdpush_sender_socket == -1)) { - error("STREAM %s [send to %s]: failed to connect", host->hostname, host->rrdpush_send_destination); + error("STREAM %s [send to %s]: failed to connect", rrdhost_hostname(host), host->rrdpush_send_destination); return 0; } - info("STREAM %s [send to %s]: initializing communication...", host->hostname, s->connected_to); + info("STREAM %s [send to %s]: initializing communication...", rrdhost_hostname(host), s->connected_to); #ifdef ENABLE_HTTPS if( netdata_client_ctx ){ @@ -370,19 +406,19 @@ if(!s->rrdpush_compression) "User-Agent: %s/%s\r\n" "Accept: */*\r\n\r\n" , host->rrdpush_send_api_key - , host->hostname - , host->registry_hostname + , rrdhost_hostname(host) + , rrdhost_registry_hostname(host) , host->machine_guid , default_rrd_update_every - , host->os - , host->timezone - , host->abbrev_timezone + , rrdhost_os(host) + , rrdhost_timezone(host) + , rrdhost_abbrev_timezone(host) , host->utc_offset , host->system_info->hops + 1 , host->system_info->ml_capable , host->system_info->ml_enabled , host->system_info->mc_version - , (host->tags) ? host->tags : "" + , rrdhost_tags(host) , s->version , (host->system_info->cloud_provider_type) ? host->system_info->cloud_provider_type : "" , (host->system_info->cloud_instance_type) ? host->system_info->cloud_instance_type : "" @@ -412,8 +448,8 @@ if(!s->rrdpush_compression) , (host->system_info->host_ram_total) ? host->system_info->host_ram_total : "" , (host->system_info->host_disk_space) ? host->system_info->host_disk_space : "" , STREAMING_PROTOCOL_VERSION - , host->program_name - , host->program_version + , rrdhost_program_name(host) + , rrdhost_program_version(host) ); http[eol] = 0x00; rrdpush_clean_encoded(&se); @@ -456,12 +492,12 @@ if(!s->rrdpush_compression) if(send_timeout(host->rrdpush_sender_socket, http, strlen(http), 0, timeout) == -1) { #endif worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT); - error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", host->hostname, s->connected_to); + error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", rrdhost_hostname(host), s->connected_to); rrdpush_sender_thread_close_socket(host); return 0; } - info("STREAM %s [send to %s]: waiting response from remote netdata...", host->hostname, s->connected_to); + info("STREAM %s [send to %s]: waiting response from remote netdata...", rrdhost_hostname(host), s->connected_to); ssize_t received; #ifdef ENABLE_HTTPS @@ -472,7 +508,7 @@ if(!s->rrdpush_compression) if(received == -1) { #endif worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT); - error("STREAM %s [send to %s]: remote netdata does not respond.", host->hostname, s->connected_to); + error("STREAM %s [send to %s]: remote netdata does not respond.", rrdhost_hostname(host), s->connected_to); rrdpush_sender_thread_close_socket(host); return 0; } @@ -482,7 +518,7 @@ if(!s->rrdpush_compression) int32_t version = (int32_t)parse_stream_version(host, http); if(version == -1) { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE); - error("STREAM %s [send to %s]: server is not replying properly (is it a netdata?).", host->hostname, s->connected_to); + error("STREAM %s [send to %s]: server is not replying properly (is it a netdata?).", rrdhost_hostname(host), s->connected_to); rrdpush_sender_thread_close_socket(host); //catch other reject reasons and force to check other destinations if (host->destination->next) @@ -490,19 +526,19 @@ if(!s->rrdpush_compression) return 0; } else if(version == -2) { - error("STREAM %s [send to %s]: remote server is the localhost for [%s].", host->hostname, s->connected_to, host->hostname); + error("STREAM %s [send to %s]: remote server is the localhost for [%s].", rrdhost_hostname(host), s->connected_to, rrdhost_hostname(host)); rrdpush_sender_thread_close_socket(host); host->destination->disabled_because_of_localhost = 1; return 0; } else if(version == -3) { - error("STREAM %s [send to %s]: remote server already receives metrics for [%s].", host->hostname, s->connected_to, host->hostname); + error("STREAM %s [send to %s]: remote server already receives metrics for [%s].", rrdhost_hostname(host), s->connected_to, rrdhost_hostname(host)); rrdpush_sender_thread_close_socket(host); host->destination->disabled_already_streaming = now_realtime_sec(); return 0; } else if(version == -4) { - error("STREAM %s [send to %s]: remote server denied access for [%s].", host->hostname, s->connected_to, host->hostname); + error("STREAM %s [send to %s]: remote server denied access for [%s].", rrdhost_hostname(host), s->connected_to, rrdhost_hostname(host)); rrdpush_sender_thread_close_socket(host); if (host->destination->next) host->destination->disabled_because_of_denied_access = 1; @@ -520,23 +556,23 @@ if(!s->rrdpush_compression) } else { //parent does not support compression or has compression disabled - debug(D_STREAM, "Stream is uncompressed! One of the agents (%s <-> %s) does not support compression OR compression is disabled.", s->connected_to, s->host->hostname); - infoerr("Stream is uncompressed! One of the agents (%s <-> %s) does not support compression OR compression is disabled.", s->connected_to, s->host->hostname); + debug(D_STREAM, "Stream is uncompressed! One of the agents (%s <-> %s) does not support compression OR compression is disabled.", s->connected_to, rrdhost_hostname(s->host)); + infoerr("Stream is uncompressed! One of the agents (%s <-> %s) does not support compression OR compression is disabled.", s->connected_to, rrdhost_hostname(s->host)); s->version = STREAM_VERSION_CLABELS; } #endif //ENABLE_COMPRESSION - info("STREAM %s [send to %s]: established communication with a parent using protocol version %d - ready to send metrics..." - , host->hostname + info("STREAM %s [send to %s]: established communication with a parent using protocol version %d" + , rrdhost_hostname(host) , s->connected_to , s->version); if(sock_setnonblock(host->rrdpush_sender_socket) < 0) - error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", host->hostname, s->connected_to); + error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", rrdhost_hostname(host), s->connected_to); if(sock_enlarge_out(host->rrdpush_sender_socket) < 0) - error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", host->hostname, s->connected_to); + error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", rrdhost_hostname(host), s->connected_to); debug(D_STREAM, "STREAM: Connected on fd %d...", host->rrdpush_sender_socket); @@ -578,7 +614,8 @@ static void attempt_to_connect(struct sender_state *state) } // TCP window is open and we have data to transmit. -void attempt_to_send(struct sender_state *s) { +static ssize_t attempt_to_send(struct sender_state *s) { + ssize_t ret = 0; rrdpush_send_labels(s->host); @@ -591,42 +628,44 @@ void attempt_to_send(struct sender_state *s) { char *chunk; size_t outstanding = cbuffer_next_unsafe(s->buffer, &chunk); debug(D_STREAM, "STREAM: Sending data. Buffer r=%zu w=%zu s=%zu, next chunk=%zu", cb->read, cb->write, cb->size, outstanding); - ssize_t ret; + #ifdef ENABLE_HTTPS SSL *conn = s->host->ssl.conn ; - if(conn && !s->host->ssl.flags) { + if(conn && !s->host->ssl.flags) ret = SSL_write(conn, chunk, outstanding); - } else { + else ret = send(s->host->rrdpush_sender_socket, chunk, outstanding, MSG_DONTWAIT); - } #else ret = send(s->host->rrdpush_sender_socket, chunk, outstanding, MSG_DONTWAIT); #endif + if (likely(ret > 0)) { cbuffer_remove_unsafe(s->buffer, ret); s->sent_bytes_on_this_connection += ret; s->sent_bytes += ret; - debug(D_STREAM, "STREAM %s [send to %s]: Sent %zd bytes", s->host->hostname, s->connected_to, ret); + debug(D_STREAM, "STREAM %s [send to %s]: Sent %zd bytes", rrdhost_hostname(s->host), s->connected_to, ret); s->last_sent_t = now_monotonic_sec(); } else if (ret == -1 && (errno == EAGAIN || errno == EINTR || errno == EWOULDBLOCK)) - debug(D_STREAM, "STREAM %s [send to %s]: unavailable after polling POLLOUT", s->host->hostname, s->connected_to); + debug(D_STREAM, "STREAM %s [send to %s]: unavailable after polling POLLOUT", rrdhost_hostname(s->host), s->connected_to); else if (ret == -1) { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SEND_ERROR); debug(D_STREAM, "STREAM: Send failed - closing socket..."); - error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", s->host->hostname, s->connected_to, s->sent_bytes_on_this_connection); + error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", rrdhost_hostname(s->host), s->connected_to, s->sent_bytes_on_this_connection); rrdpush_sender_thread_close_socket(s->host); } - else { + else debug(D_STREAM, "STREAM: send() returned 0 -> no error but no transmission"); - } netdata_mutex_unlock(&s->mutex); netdata_thread_enable_cancelability(); + + return ret; } -void attempt_read(struct sender_state *s) { -int ret; +static ssize_t attempt_read(struct sender_state *s) { + ssize_t ret = 0; + #ifdef ENABLE_HTTPS if (s->host->ssl.conn && !s->host->stream_ssl.flags) { ERR_clear_error(); @@ -634,44 +673,46 @@ int ret; ret = SSL_read(s->host->ssl.conn, s->read_buffer, desired); if (ret > 0 ) { s->read_len += ret; - return; + return ret; } int sslerrno = SSL_get_error(s->host->ssl.conn, desired); if (sslerrno == SSL_ERROR_WANT_READ || sslerrno == SSL_ERROR_WANT_WRITE) - return; + return ret; worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR); u_long err; char buf[256]; while ((err = ERR_get_error()) != 0) { ERR_error_string_n(err, buf, sizeof(buf)); - error("STREAM %s [send to %s] ssl error: %s", s->host->hostname, s->connected_to, buf); + error("STREAM %s [send to %s] ssl error: %s", rrdhost_hostname(s->host), s->connected_to, buf); } error("Restarting connection"); rrdpush_sender_thread_close_socket(s->host); - return; + return ret; } #endif ret = recv(s->host->rrdpush_sender_socket, s->read_buffer + s->read_len, sizeof(s->read_buffer) - s->read_len - 1,MSG_DONTWAIT); - if (ret>0) { + if (ret > 0) { s->read_len += ret; - return; + return ret; } - debug(D_STREAM, "Socket was POLLIN, but req %zu bytes gave %d", sizeof(s->read_buffer) - s->read_len - 1, ret); + debug(D_STREAM, "Socket was POLLIN, but req %zu bytes gave %zd", sizeof(s->read_buffer) - s->read_len - 1, ret); - if (ret<0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) - return; + if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) + return ret; - if (ret==0) { + if (ret == 0) { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_PARENT_CLOSED); - error("STREAM %s [send to %s]: connection closed by far end. Restarting connection", s->host->hostname, s->connected_to); + error("STREAM %s [send to %s]: connection closed by far end. Restarting connection", rrdhost_hostname(s->host), s->connected_to); } else { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_RECEIVE_ERROR); - error("STREAM %s [send to %s]: error during receive (%d). Restarting connection", s->host->hostname, s->connected_to, ret); + error("STREAM %s [send to %s]: error during receive (%zd). Restarting connection", rrdhost_hostname(s->host), s->connected_to, ret); } rrdpush_sender_thread_close_socket(s->host); + + return ret; } // This is just a placeholder until the gap filling state machine is inserted @@ -680,7 +721,7 @@ void execute_commands(struct sender_state *s) { *end = 0; while( starthost->hostname, s->connected_to, start); + info("STREAM %s [send to %s] received command over connection: %s", rrdhost_hostname(s->host), s->connected_to, start); start = newline+1; } if (startsender_state->mutex); + size_t outstanding = cbuffer_available_size_unsafe(thread_data->sender_state->host->sender->buffer); + netdata_mutex_unlock(&thread_data->sender_state->mutex); + return outstanding; +} + +static void rrdpush_queue_incremental_definitions(struct rrdpush_sender_thread_data *thread_data) { + + while(__atomic_load_n(&thread_data->host->rrdpush_sender_connected, __ATOMIC_SEQ_CST) + && thread_data->sending_definitions_status != SENDING_DEFINITIONS_DONE + && cbuffer_available_bytes_with_lock(thread_data) > (thread_data->sender_state->buffer->max_size / 2)) { + + if(thread_data->sending_definitions_status == SENDING_DEFINITIONS_RESTART) + info("STREAM %s [send to %s]: sending metric definitions...", rrdhost_hostname(thread_data->host), thread_data->sender_state->connected_to); + + bool more_defs_available = rrdpush_incremental_transmission_of_chart_definitions( + thread_data->sender_state->host, &thread_data->dictfe, + thread_data->sending_definitions_status == SENDING_DEFINITIONS_RESTART, false); + + if (unlikely(!more_defs_available)) { + thread_data->sending_definitions_status = SENDING_DEFINITIONS_DONE; + info("STREAM %s [send to %s]: sending metric definitions finished.", rrdhost_hostname(thread_data->host), thread_data->sender_state->connected_to); + } + else + thread_data->sending_definitions_status = SENDING_DEFINITIONS_CONTINUE; + } +} static void rrdpush_sender_thread_cleanup_callback(void *ptr) { + struct rrdpush_sender_thread_data *data = ptr; worker_unregister(); - RRDHOST *host = (RRDHOST *)ptr; + RRDHOST *host = data->host; + + rrdpush_incremental_transmission_of_chart_definitions(host, &data->dictfe, false, true); netdata_mutex_lock(&host->sender->mutex); - info("STREAM %s [send]: sending thread cleans up...", host->hostname); + info("STREAM %s [send]: sending thread cleans up...", rrdhost_hostname(host)); rrdpush_sender_thread_close_socket(host); @@ -713,15 +796,17 @@ static void rrdpush_sender_thread_cleanup_callback(void *ptr) { } if(!host->rrdpush_sender_join) { - info("STREAM %s [send]: sending thread detaches itself.", host->hostname); + info("STREAM %s [send]: sending thread detaches itself.", rrdhost_hostname(host)); netdata_thread_detach(netdata_thread_self()); } host->rrdpush_sender_spawn = 0; - info("STREAM %s [send]: sending thread now exits.", host->hostname); + info("STREAM %s [send]: sending thread now exits.", rrdhost_hostname(host)); netdata_mutex_unlock(&host->sender->mutex); + + freez(data); } void sender_init(RRDHOST *parent) @@ -749,7 +834,7 @@ void *rrdpush_sender_thread(void *ptr) { !*s->host->rrdpush_send_destination || !s->host->rrdpush_send_api_key || !*s->host->rrdpush_send_api_key) { error("STREAM %s [send]: thread created (task id %d), but host has streaming disabled.", - s->host->hostname, s->task_id); + rrdhost_hostname(s->host), s->task_id); return NULL; } @@ -760,7 +845,7 @@ void *rrdpush_sender_thread(void *ptr) { } #endif - info("STREAM %s [send]: thread created (task id %d)", s->host->hostname, s->task_id); + info("STREAM %s [send]: thread created (task id %d)", rrdhost_hostname(s->host), s->task_id); s->timeout = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "timeout seconds", 60); s->default_port = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "default port", 19999); @@ -774,9 +859,10 @@ void *rrdpush_sender_thread(void *ptr) { remote_clock_resync_iterations); // TODO: REMOVE FOR SLEW / GAPFILLING // initialize rrdpush globals + rrdhost_flag_clear(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS); __atomic_clear(&s->host->rrdpush_sender_connected, __ATOMIC_SEQ_CST); if(pipe(s->host->rrdpush_sender_pipe) == -1) { - error("STREAM %s [send]: cannot create required pipe. DISABLING STREAMING THREAD", s->host->hostname); + error("STREAM %s [send]: cannot create required pipe. DISABLING STREAMING THREAD", rrdhost_hostname(s->host)); return NULL; } s->version = STREAMING_PROTOCOL_CURRENT_VERSION; @@ -808,7 +894,17 @@ void *rrdpush_sender_thread(void *ptr) { worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION, "disconnect no compression"); worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE, "disconnect bad handshake"); - netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, s->host); + worker_register_job_custom_metric(WORKER_SENDER_JOB_BUFFER_RATIO, "used buffer ratio", "%", WORKER_METRIC_ABSOLUTE); + worker_register_job_custom_metric(WORKER_SENDER_JOB_BYTES_RECEIVED, "bytes received", "bytes/s", WORKER_METRIC_INCREMENTAL); + worker_register_job_custom_metric(WORKER_SENDER_JOB_BYTES_SENT, "bytes sent", "bytes/s", WORKER_METRIC_INCREMENTAL); + + struct rrdpush_sender_thread_data *thread_data = callocz(1, sizeof(struct rrdpush_sender_thread_data)); + thread_data->sender_state = s; + thread_data->host = s->host; + thread_data->sending_definitions_status = SENDING_DEFINITIONS_RESTART; + + netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, thread_data); + for(; s->host->rrdpush_send_enabled && !netdata_exit ;) { // check for outstanding cancellation requests netdata_thread_testcancel(); @@ -816,6 +912,8 @@ void *rrdpush_sender_thread(void *ptr) { // The connection attempt blocks (after which we use the socket in nonblocking) if(unlikely(s->host->rrdpush_sender_socket == -1)) { worker_is_busy(WORKER_SENDER_JOB_CONNECT); + thread_data->sending_definitions_status = SENDING_DEFINITIONS_RESTART; + rrdhost_flag_clear(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS); s->overflow = 0; s->read_len = 0; s->buffer->read = 0; @@ -828,17 +926,27 @@ void *rrdpush_sender_thread(void *ptr) { sender_commit(s); } rrdpush_claimed_id(s->host); + + // TO PUSH METRICS WITH DEFINITIONS: + //if(unlikely(s->host->rrdpush_sender_socket != -1 && __atomic_load_n(&s->host->rrdpush_sender_connected, __ATOMIC_SEQ_CST))) { + // thread_data->sending_definitions_status = SENDING_DEFINITIONS_DONE; + // rrdhost_flag_set(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS); + //} + continue; } // If the TCP window never opened then something is wrong, restart connection if(unlikely(now_monotonic_sec() - s->last_sent_t > s->timeout)) { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT); - error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", s->host->hostname, s->connected_to, s->timeout, s->sent_bytes_on_this_connection, s->send_attempts); + error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", rrdhost_hostname(s->host), s->connected_to, s->timeout, s->sent_bytes_on_this_connection, s->send_attempts); rrdpush_sender_thread_close_socket(s->host); continue; } + if(unlikely(thread_data->sending_definitions_status != SENDING_DEFINITIONS_DONE)) + rrdpush_queue_incremental_definitions(thread_data); + worker_is_idle(); // Wait until buffer opens in the socket or a rrdset_done_push wakes us @@ -847,16 +955,27 @@ void *rrdpush_sender_thread(void *ptr) { fds[Socket].fd = s->host->rrdpush_sender_socket; netdata_mutex_lock(&s->mutex); - char *chunk; - size_t outstanding = cbuffer_next_unsafe(s->host->sender->buffer, &chunk); - chunk = NULL; // Do not cache pointer outside of region - could be invalidated + size_t outstanding = cbuffer_next_unsafe(s->host->sender->buffer, NULL); + size_t available = cbuffer_available_size_unsafe(s->host->sender->buffer); netdata_mutex_unlock(&s->mutex); + + worker_set_metric(WORKER_SENDER_JOB_BUFFER_RATIO, (NETDATA_DOUBLE)(s->host->sender->buffer->max_size - available) * 100.0 / (NETDATA_DOUBLE)s->host->sender->buffer->max_size); + if(outstanding) { s->send_attempts++; fds[Socket].events = POLLIN | POLLOUT; } else { fds[Socket].events = POLLIN; + + if(unlikely(thread_data->sending_definitions_status == SENDING_DEFINITIONS_DONE + && __atomic_load_n(&s->host->rrdpush_sender_connected, __ATOMIC_SEQ_CST) + && !rrdhost_flag_check(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS) + )) { + // let the data collection threads know we are ready to push metrics + rrdhost_flag_set(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS); + info("STREAM %s [send to %s]: enabling metrics streaming...", rrdhost_hostname(s->host), s->connected_to); + } } int retval = poll(fds, 2, 1000); @@ -874,7 +993,7 @@ void *rrdpush_sender_thread(void *ptr) { // Only errors from poll() are internal, but try restarting the connection if(unlikely(retval == -1)) { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_POLL_ERROR); - error("STREAM %s [send to %s]: failed to poll(). Closing socket.", s->host->hostname, s->connected_to); + error("STREAM %s [send to %s]: failed to poll(). Closing socket.", rrdhost_hostname(s->host), s->connected_to); rrdpush_sender_thread_close_socket(s->host); continue; } @@ -884,28 +1003,34 @@ void *rrdpush_sender_thread(void *ptr) { worker_is_busy(WORKER_SENDER_JOB_PIPE_READ); debug(D_STREAM, "STREAM: Data added to send buffer (current buffer chunk %zu bytes)...", outstanding); - char buffer[1000 + 1]; - if (read(s->host->rrdpush_sender_pipe[PIPE_READ], buffer, 1000) == -1) - error("STREAM %s [send to %s]: cannot read from internal pipe.", s->host->hostname, s->connected_to); + char buffer[10000 + 1]; + if (read(s->host->rrdpush_sender_pipe[PIPE_READ], buffer, 10000) == -1) + error("STREAM %s [send to %s]: cannot read from internal pipe.", rrdhost_hostname(s->host), s->connected_to); } // Read as much as possible to fill the buffer, split into full lines for execution. if (fds[Socket].revents & POLLIN) { worker_is_busy(WORKER_SENDER_JOB_SOCKET_RECEIVE); - attempt_read(s); + ssize_t bytes = attempt_read(s); + if(bytes > 0) + worker_set_metric(WORKER_SENDER_JOB_BYTES_RECEIVED, bytes); } - worker_is_busy(WORKER_SENDER_JOB_EXECUTE); - execute_commands(s); + if(unlikely(s->read_len)) { + worker_is_busy(WORKER_SENDER_JOB_EXECUTE); + execute_commands(s); + } // If we have data and have seen the TCP window open then try to close it by a transmission. - if (outstanding && fds[Socket].revents & POLLOUT) { + if(likely(outstanding && fds[Socket].revents & POLLOUT)) { worker_is_busy(WORKER_SENDER_JOB_SOCKET_SEND); - attempt_to_send(s); + ssize_t bytes = attempt_to_send(s); + if(bytes > 0) + worker_set_metric(WORKER_SENDER_JOB_BYTES_SENT, bytes); } // TODO-GAPS - why do we only check this on the socket, not the pipe? - if (outstanding) { + if(outstanding) { char *error = NULL; if (unlikely(fds[Socket].revents & POLLERR)) error = "socket reports errors (POLLERR)"; @@ -915,18 +1040,18 @@ void *rrdpush_sender_thread(void *ptr) { error = "connection is invalid (POLLNVAL)"; if(unlikely(error)) { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SOCKER_ERROR); - error("STREAM %s [send to %s]: restart stream because %s - %zu bytes transmitted.", s->host->hostname, + error("STREAM %s [send to %s]: restart stream because %s - %zu bytes transmitted.", rrdhost_hostname(s->host), s->connected_to, error, s->sent_bytes_on_this_connection); rrdpush_sender_thread_close_socket(s->host); } } // protection from overflow - if (s->overflow) { + if(unlikely(s->overflow)) { worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW); errno = 0; - error("STREAM %s [send to %s]: buffer full (%zu-bytes) after %zu bytes. Restarting connection", - s->host->hostname, s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection); + error("STREAM %s [send to %s]: buffer full (allocated %zu bytes) after sending %zu bytes. Restarting connection", + rrdhost_hostname(s->host), s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection); rrdpush_sender_thread_close_socket(s->host); } } diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c index 00b4ad650a..c4d9ced5f0 100644 --- a/web/api/badges/web_buffer_svg.c +++ b/web/api/badges/web_buffer_svg.c @@ -1020,19 +1020,19 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u label = dim; } else - label = st->name; + label = rrdset_name(st); } if(!units) { if(alarm) { if(rc->units) - units = rc->units; + units = rrdcalc_units(rc); else units = ""; } else if(options & RRDR_OPTION_PERCENTAGE) units = "%"; else - units = st->units; + units = rrdset_units(st); } debug(D_WEB_CLIENT, "%llu: API command 'badge.svg' for chart '%s', alarm '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', options '0x%08x'" diff --git a/web/api/exporters/shell/allmetrics_shell.c b/web/api/exporters/shell/allmetrics_shell.c index 615aab43cd..184f44b188 100644 --- a/web/api/exporters/shell/allmetrics_shell.c +++ b/web/api/exporters/shell/allmetrics_shell.c @@ -30,14 +30,14 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ // for each chart RRDSET *st; rrdset_foreach_read(st, host) { - if (filter && !simple_pattern_matches(filter, st->name)) + if (filter && !simple_pattern_matches(filter, rrdset_name(st))) continue; NETDATA_DOUBLE total = 0.0; char chart[SHELL_ELEMENT_MAX + 1]; - shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX); + shell_name_copy(chart, st->name?rrdset_name(st):rrdset_id(st), SHELL_ELEMENT_MAX); - buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name); + buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", rrdset_id(st), rrdset_name(st)); if(rrdset_is_available_for_viewers(st)) { rrdset_rdlock(st); @@ -46,23 +46,23 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ rrddim_foreach_read(rd, st) { if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { char dimension[SHELL_ELEMENT_MAX + 1]; - shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX); + shell_name_copy(dimension, rd->name?rrddim_name(rd):rrddim_id(rd), SHELL_ELEMENT_MAX); NETDATA_DOUBLE n = rd->last_stored_value; if(isnan(n) || isinf(n)) - buffer_sprintf(wb, "NETDATA_%s_%s=\"\" # %s\n", chart, dimension, st->units); + buffer_sprintf(wb, "NETDATA_%s_%s=\"\" # %s\n", chart, dimension, rrdset_units(st)); else { if(rd->multiplier < 0 || rd->divisor < 0) n = -n; n = roundndd(n); if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n; - buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, dimension, n, st->units); + buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, dimension, n, rrdset_units(st)); } } } total = roundndd(total); - buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, total, st->units); + buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, total, rrdset_units(st)); rrdset_unlock(st); } } @@ -70,22 +70,22 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n"); RRDCALC *rc; - for(rc = host->alarms; rc ;rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if(!rc->rrdset) continue; char chart[SHELL_ELEMENT_MAX + 1]; - shell_name_copy(chart, rc->rrdset->name?rc->rrdset->name:rc->rrdset->id, SHELL_ELEMENT_MAX); + shell_name_copy(chart, rc->rrdset->name?rrdset_name(rc->rrdset):rrdset_id(rc->rrdset), SHELL_ELEMENT_MAX); char alarm[SHELL_ELEMENT_MAX + 1]; - shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX); + shell_name_copy(alarm, rrdcalc_name(rc), SHELL_ELEMENT_MAX); NETDATA_DOUBLE n = rc->value; if(isnan(n) || isinf(n)) - buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\" # %s\n", chart, alarm, rc->units); + buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\" # %s\n", chart, alarm, rrdcalc_units(rc)); else { n = roundndd(n); - buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, alarm, n, rc->units); + buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, alarm, n, rrdcalc_units(rc)); } buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status)); @@ -110,7 +110,7 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s // for each chart RRDSET *st; rrdset_foreach_read(st, host) { - if (filter && !(simple_pattern_matches(filter, st->id) || simple_pattern_matches(filter, st->name))) + if (filter && !(simple_pattern_matches(filter, rrdset_id(st)) || simple_pattern_matches(filter, rrdset_name(st)))) continue; if(rrdset_is_available_for_viewers(st)) { @@ -127,11 +127,11 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s "\t\t\"last_updated\": %"PRId64",\n" "\t\t\"dimensions\": {", chart_counter ? "," : "", - st->id, - st->name, - st->family, - st->context, - st->units, + rrdset_id(st), + rrdset_name(st), + rrdset_family(st), + rrdset_context(st), + rrdset_units(st), (int64_t)rrdset_last_entry_t_nolock(st)); chart_counter++; @@ -148,8 +148,8 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s "\t\t\t\t\"name\": \"%s\",\n" "\t\t\t\t\"value\": ", dimension_counter ? "," : "", - rd->id, - rd->name); + rrddim_id(rd), + rrddim_name(rd)); if(isnan(rd->last_stored_value)) buffer_strcat(wb, "null"); diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c index 4325b65301..73e4247f4e 100644 --- a/web/api/formatters/charts2json.c +++ b/web/api/formatters/charts2json.c @@ -57,11 +57,11 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived ",\n\t\"memory_mode\": \"%s\"" ",\n\t\"custom_info\": \"%s\"" ",\n\t\"charts\": {" - , host->hostname - , host->program_version + , rrdhost_hostname(host) + , rrdhost_program_version(host) , get_release_channel() - , host->os - , host->timezone + , rrdhost_os(host) + , rrdhost_timezone(host) , host->rrd_update_every , host->rrd_history_entries , rrd_memory_mode_name(host->rrd_memory_mode) @@ -74,7 +74,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) { if(c) buffer_strcat(wb, ","); buffer_strcat(wb, "\n\t\t\""); - buffer_strcat(wb, st->id); + buffer_strcat(wb, rrdset_id(st)); buffer_strcat(wb, "\": "); rrdset2json(st, wb, &dimensions, &memory, skip_volatile); @@ -84,7 +84,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived } RRDCALC *rc; - for(rc = host->alarms; rc ; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if(rc->rrdset) alarms++; } @@ -117,7 +117,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived "\n\t\t\t\"hostname\": \"%s\"" "\n\t\t}" , (found > 0) ? "," : "" - , h->hostname + , rrdhost_hostname(h) ); found++; @@ -131,7 +131,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived , "\n\t\t{" "\n\t\t\t\"hostname\": \"%s\"" "\n\t\t}" - , host->hostname + , rrdhost_hostname(host) ); } @@ -141,8 +141,8 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived // generate collectors list for the api/v1/info call struct collector { - char *plugin; - char *module; + const char *plugin; + const char *module; }; struct array_printer { @@ -176,8 +176,8 @@ void chartcollectors2json(RRDHOST *host, BUFFER *wb) { rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { struct collector col = { - .plugin = st->plugin_name ? st->plugin_name : "", - .module = st->module_name ? st->module_name : "" + .plugin = rrdset_plugin_name(st), + .module = rrdset_module_name(st) }; sprintf(name, "%s:%s", col.plugin, col.module); dictionary_set(dict, name, &col, sizeof(struct collector)); diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c index 6d87ca3748..14dad0cb8a 100644 --- a/web/api/formatters/csv/csv.c +++ b/web/api/formatters/csv/csv.c @@ -23,7 +23,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const } buffer_strcat(wb, separator); if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - buffer_strcat(wb, d->name); + buffer_strcat(wb, rrddim_name(d)); if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); i++; } diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c index 6f07b9aa43..ae1e55b7a9 100644 --- a/web/api/formatters/json/json.c +++ b/web/api/formatters/json/json.c @@ -121,7 +121,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; buffer_fast_strcat(wb, pre_label, pre_label_len); - buffer_strcat(wb, rd->name); + buffer_strcat(wb, rrddim_name(rd)); // buffer_strcat(wb, "."); // buffer_strcat(wb, rd->rrdset->name); buffer_fast_strcat(wb, post_label, post_label_len); @@ -253,7 +253,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct buffer_fast_strcat(wb, pre_value, pre_value_len); if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) - buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq); + buffer_sprintf(wb, "%s%s%s: ", kq, rrddim_name(rd), kq); if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) { if(unlikely(options & RRDR_OPTION_NULL2ZERO)) diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index 04cace2fb3..a07ee17446 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -79,8 +79,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS " %sgroup%s: %s%s%s,\n" " %soptions%s: %s" , kq, kq - , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq - , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq + , kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_id(r->st), sq + , kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_name(r->st), sq , kq, kq, r->update_every , kq, kq, r->st->update_every , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t_nolock(r->st)) @@ -103,13 +103,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS if(i) buffer_strcat(wb, ", "); buffer_strcat(wb, sq); - buffer_strcat(wb, rd->name); + buffer_strcat(wb, rrddim_name(rd)); buffer_strcat(wb, sq); i++; } if(!i) { #ifdef NETDATA_INTERNAL_CHECKS - error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options); + error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", rrdset_id(r->st), r->d, options); #endif rows = 0; buffer_strcat(wb, sq); @@ -127,7 +127,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS if(i) buffer_strcat(wb, ", "); buffer_strcat(wb, sq); - buffer_strcat(wb, rd->id); + buffer_strcat(wb, rrddim_id(rd)); buffer_strcat(wb, sq); i++; } @@ -149,8 +149,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { - snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name); - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name); + snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrddim_id(rd), rrddim_name(rd)); + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrddim_id(rd), rrddim_name(rd)); dictionary_set(dict, name, output, len+1); } dictionary_walkthrough_read(dict, value_list_output, &co); @@ -160,8 +160,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq); dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name); - snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name); + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset)); + snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset)); dictionary_set(dict, name, output, len + 1); } @@ -198,7 +198,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS if (i) buffer_strcat(wb, ", "); buffer_strcat(wb, sq); - buffer_strcat(wb, rd->rrdset->id); + buffer_strcat(wb, rrdset_id(rd->rrdset)); buffer_strcat(wb, sq); i++; } diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 7aa478d95a..b18abe59cb 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -8,13 +8,15 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc if (unlikely(!temp_rd)) return; - onewayalloc_freez(owa, (char *)temp_rd->id); + string_freez(temp_rd->id); + string_freez(temp_rd->name); if (unlikely(archive_mode)) { temp_rd->rrdset->counter--; if (!temp_rd->rrdset->counter) { - onewayalloc_freez(owa, (char *)temp_rd->rrdset->name); - onewayalloc_freez(owa, temp_rd->rrdset->context); + string_freez(temp_rd->rrdset->id); + string_freez(temp_rd->rrdset->name); + string_freez(temp_rd->rrdset->context); onewayalloc_freez(owa, temp_rd->rrdset); } } @@ -111,8 +113,8 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis rrddim_foreach_read(rd1, st) { RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM)); - rd->id = onewayalloc_strdupz(owa, rd1->id); - rd->name = onewayalloc_strdupz(owa, rd1->name); + rd->id = string_dup(rd1->id); + rd->name = string_dup(rd1->name); for(int tier = 0; tier < storage_tiers ;tier++) { if(rd1->tiers[tier]) rd->tiers[tier] = onewayalloc_memdupz(owa, rd1->tiers[tier], sizeof(*rd->tiers[tier])); diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c index de8d87bae5..9e81389fdf 100644 --- a/web/api/formatters/rrdset2json.c +++ b/web/api/formatters/rrdset2json.c @@ -45,18 +45,18 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor "\t\t\t\"units\": \"%s\",\n" "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n" "\t\t\t\"chart_type\": \"%s\",\n", - st->id, - st->name, - st->type, - st->family, - st->context, - st->title, - st->name, + rrdset_id(st), + rrdset_name(st), + rrdset_type(st), + rrdset_family(st), + rrdset_context(st), + rrdset_title(st), + rrdset_name(st), st->priority, - st->plugin_name ? st->plugin_name : "", - st->module_name ? st->module_name : "", - st->units, - st->name, + rrdset_plugin_name(st), + rrdset_module_name(st), + rrdset_units(st), + rrdset_name(st), rrdset_type_name(st->chart_type)); if (likely(!skip_volatile)) @@ -98,9 +98,9 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_strcat(wb, ",\n\t\t\t\t\""); else buffer_strcat(wb, "\t\t\t\t\""); - buffer_strcat_jsonescape(wb, rd->id); + buffer_strcat_jsonescape(wb, rrddim_id(rd)); buffer_strcat(wb, "\": { \"name\": \""); - buffer_strcat_jsonescape(wb, rd->name); + buffer_strcat_jsonescape(wb, rrddim_name(rd)); buffer_strcat(wb, "\" }"); dimensions++; @@ -121,7 +121,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n"); size_t alarms = 0; RRDCALC *rc; - for (rc = st->alarms; rc; rc = rc->rrdset_next) { + foreach_rrdcalc_in_rrdset(st, rc) { buffer_sprintf( wb, "%s" @@ -131,7 +131,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor "\t\t\t\t\t\"units\": \"%s\",\n" "\t\t\t\t\t\"update_every\": %d\n" "\t\t\t\t}", - (alarms) ? ",\n" : "", rc->name, rc->id, rrdcalc_status2string(rc->status), rc->units, + (alarms) ? ",\n" : "", rrdcalc_name(rc), rc->id, rrdcalc_status2string(rc->status), rrdcalc_units(rc), rc->update_every); alarms++; diff --git a/web/api/queries/query.c b/web/api/queries/query.c index d776f6d11c..300a4429ce 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -665,11 +665,11 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)); + if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return; + if (should_lock) rrdset_check_rdlock(r->st); - if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return; - int match_ids = 0, match_names = 0; if(unlikely(options & RRDR_OPTION_MATCH_IDS)) @@ -685,8 +685,8 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, RRDDIM *d; long c, dims_selected = 0, dims_not_hidden_not_zero = 0; for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next) { - if( (match_ids && simple_pattern_matches(pattern, d->id)) - || (match_names && simple_pattern_matches(pattern, d->name)) + if( (match_ids && simple_pattern_matches(pattern, rrddim_id(d))) + || (match_names && simple_pattern_matches(pattern, rrddim_name(d))) ) { r->od[c] |= RRDR_DIMENSION_SELECTED; if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) r->od[c] &= ~RRDR_DIMENSION_HIDDEN; @@ -738,11 +738,11 @@ static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) { internal_error(rrdr_line >= r->n, "QUERY: requested to step above RRDR size for chart '%s'", - r->st->name); + rrdset_name(r->st)); internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t, "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'", - (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name); + (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, rrdset_name(r->st)); // save the time r->t[rrdr_line] = t; @@ -768,7 +768,7 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, internal_error(true, "QUERY: NULL dimension - invalid params to tier calculation"); else internal_error(true, "QUERY: chart '%s' dimension '%s' invalid params to tier calculation", - (rd->rrdset)?rd->rrdset->name:"unknown", rd->name); + (rd->rrdset)?rrdset_name(rd->rrdset):"unknown", rrddim_name(rd)); return 0; } @@ -782,7 +782,7 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, for(int tier = 0; tier < storage_tiers ; tier++) { if(unlikely(!rd->tiers[tier])) { internal_error(true, "QUERY: tier %d of chart '%s' dimension '%s' not initialized", - tier, rd->rrdset->name, rd->name); + tier, rrdset_name(rd->rrdset), rrddim_name(rd)); // buffer_free(wb); return 0; } @@ -799,7 +799,7 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, int update_every = (int)rd->tiers[tier]->tier_grouping * (int)rd->update_every; if(unlikely(update_every == 0)) { internal_error(true, "QUERY: update_every of tier %d for chart '%s' dimension '%s' is zero. tg = %d, ue = %d", - tier, rd->rrdset->name, rd->name, rd->tiers[tier]->tier_grouping, rd->update_every); + tier, rrdset_name(rd->rrdset), rrddim_name(rd), rd->tiers[tier]->tier_grouping, rd->update_every); // buffer_free(wb); return 0; } @@ -851,7 +851,7 @@ static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t aft if(!st->dimensions->tiers[best_tier]) { internal_error( true, - "QUERY: tier %d on chart '%s', is not initialized", best_tier, st->name); + "QUERY: tier %d on chart '%s', is not initialized", best_tier, rrdset_name(st)); } else { ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every; @@ -859,7 +859,7 @@ static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t aft internal_error( true, "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d", - st->name, st->dimensions->tiers[best_tier]->tier_grouping, st->update_every); + rrdset_name(st), st->dimensions->tiers[best_tier]->tier_grouping, st->update_every); ret = st->update_every; } @@ -1259,7 +1259,7 @@ static inline void rrd2rrdr_do_dimension( // check if the db is giving us zero duration points if(unlikely(new_point.start_time == new_point.end_time)) { internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal", - rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time); + rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time); new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every); } @@ -1267,7 +1267,7 @@ static inline void rrd2rrdr_do_dimension( // check if the db is advancing the query if(unlikely(new_point.end_time <= last1_point.end_time)) { internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld", - rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time, + rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time, last1_point.id, last1_point.end_time, now_start_time, now_end_time); count_same_end_time++; @@ -1295,7 +1295,7 @@ static inline void rrd2rrdr_do_dimension( // we only log if this is not point 1 internal_error(new_point.end_time < after_wanted && new_point.id > 1, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)", - rd->rrdset->name, rd->name, + rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted); @@ -1339,7 +1339,7 @@ static inline void rrd2rrdr_do_dimension( internal_error(current_point.id > 0 && last1_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time, "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld," " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point1 in this query.", - rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, + rrdset_name(rd->rrdset), rrddim_name(rd), after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, current_point.id, current_point.start_time, current_point.end_time, now_end_time); } else if(likely(now_end_time <= last1_point.end_time)) { @@ -1350,7 +1350,7 @@ static inline void rrd2rrdr_do_dimension( internal_error(current_point.id > 0 && last2_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time, "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld," " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point2 in this query.", - rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, + rrdset_name(rd->rrdset), rrddim_name(rd), after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, current_point.id, current_point.start_time, current_point.end_time, now_end_time); } else { @@ -1425,7 +1425,7 @@ static inline void rrd2rrdr_do_dimension( internal_error((long)points_added != points_wanted, "QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).", - r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read); + rrdset_name(r->st), rrddim_name(rd), (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read); } // ---------------------------------------------------------------------------- @@ -1518,7 +1518,7 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r //"slot (after: %zu, before: %zu, delta: %zu), " "points (got: %ld, want: %ld, req: %ld, db: %ld), " "%s" - , r->st->name + , rrdset_name(r->st) , r->st->update_every // grouping @@ -1733,7 +1733,7 @@ RRDR *rrd2rrdr( rrdset_unlock(st); if(first_entry_t == 0 || last_entry_t == 0) { - internal_error(true, "QUERY: chart without data detected on '%s'", st->name); + internal_error(true, "QUERY: chart without data detected on '%s'", rrdset_name(st)); query_debug_log_free(); return NULL; } @@ -1932,13 +1932,13 @@ RRDR *rrd2rrdr( RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list); if(unlikely(!r)) { internal_error(true, "QUERY: cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", - st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); + rrdset_id(st), (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); return NULL; } if(unlikely(!r->d || !points_wanted)) { internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", - st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted); + rrdset_id(st), (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted); return r; } @@ -1977,10 +1977,9 @@ RRDR *rrd2rrdr( if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) rrdset_check_rdlock(st); - if(dimensions) + if(dimensions && *dimensions) rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list); - query_debug_log_fin(); // ------------------------------------------------------------------------- @@ -2024,21 +2023,21 @@ RRDR *rrd2rrdr( else { if(r->after != max_after) { internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)max_after, rd->name, (size_t)r->after); + rrdset_name(st), (size_t)max_after, rrddim_name(rd), (size_t)r->after); r->after = (r->after > max_after) ? r->after : max_after; } if(r->before != min_before) { internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)min_before, rd->name, (size_t)r->before); + rrdset_name(st), (size_t)min_before, rrddim_name(rd), (size_t)r->before); r->before = (r->before < min_before) ? r->before : min_before; } if(r->rows != max_rows) { internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)max_rows, rd->name, (size_t)r->rows); + rrdset_name(st), (size_t)max_rows, rrddim_name(rd), (size_t)r->rows); r->rows = (r->rows > max_rows) ? r->rows : max_rows; } diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c index 97a00f91cf..5ff997a26b 100644 --- a/web/api/queries/weights.c +++ b/web/api/queries/weights.c @@ -121,14 +121,14 @@ static void register_result(DICTIONARY *results, struct register_result t = { .flags = flags, .st = st, - .chart_id = st->id, - .context = st->context, - .dim_name = d->name, + .chart_id = rrdset_id(st), + .context = rrdset_context(st), + .dim_name = rrddim_name(d), .value = v }; char buf[5000 + 1]; - snprintfz(buf, 5000, "%s:%s", st->id, d->name); + snprintfz(buf, 5000, "%s:%s", rrdset_id(st), rrddim_name(d)); dictionary_set(results, buf, &t, sizeof(struct register_result)); } @@ -541,7 +541,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, group_time, options, NULL, context_param_list, group_options, timeout, tier); if(!high_rrdr) { - info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name); + info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", rrdset_name(st)); goto cleanup; } @@ -551,11 +551,11 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, stats->db_points += high_rrdr->internal.db_points_read; stats->result_points += high_rrdr->internal.result_points_generated; if(!high_rrdr->d) { - info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name); + info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", rrdset_name(st)); goto cleanup; } if(high_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) { - info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", st->name); + info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", rrdset_name(st)); goto cleanup; } int high_points = rrdr_rows(high_rrdr); @@ -571,7 +571,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, group_time, options, NULL, context_param_list, group_options, (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), tier); if(!base_rrdr) { - info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name); + info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", rrdset_name(st)); goto cleanup; } @@ -581,15 +581,15 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, stats->db_points += base_rrdr->internal.db_points_read; stats->result_points += base_rrdr->internal.result_points_generated; if(!base_rrdr->d) { - info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name); + info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", rrdset_name(st)); goto cleanup; } if (base_rrdr->d != high_rrdr->d) { - info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", st->name); + info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", rrdset_name(st)); goto cleanup; } if(base_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) { - info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", st->name); + info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", rrdset_name(st)); goto cleanup; } int base_points = rrdr_rows(base_rrdr); @@ -605,7 +605,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, // for each dimension RRDDIM *d; int i; - for(i = 0, d = base_rrdr->st->dimensions ; d && i < base_rrdr->d; i++, d = d->next) { + for(i = 0, d = base_rrdr->st->dimensions; d && i < base_rrdr->d; i++, d = d->next) { // skip the not evaluated ones if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN)) @@ -692,7 +692,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, NETDATA_DOUBLE baseline_average = NAN; NETDATA_DOUBLE base_anomaly_rate = 0; value_is_null = 1; - ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1, + ret = rrdset2value_api_v1(st, NULL, &baseline_average, rrddim_id(d), 1, baseline_after, baseline_before, group, group_options, group_time, options, NULL, NULL, @@ -709,7 +709,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, NETDATA_DOUBLE highlight_average = NAN; NETDATA_DOUBLE high_anomaly_rate = 0; value_is_null = 1; - ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1, + ret = rrdset2value_api_v1(st, NULL, &highlight_average, rrddim_id(d), 1, after, before, group, group_options, group_time, options, NULL, NULL, @@ -734,7 +734,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, char highlighted_countif_options[50 + 1]; snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average); - ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1, + ret = rrdset2value_api_v1(st, NULL, &highlight_countif, rrddim_id(d), 1, after, before, RRDR_GROUPING_COUNTIF,highlighted_countif_options, group_time, options, @@ -803,7 +803,7 @@ static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results, NETDATA_DOUBLE average = NAN; NETDATA_DOUBLE anomaly_rate = 0; value_is_null = 1; - ret = rrdset2value_api_v1(st, NULL, &average, d->id, 1, + ret = rrdset2value_api_v1(st, NULL, &average, rrddim_id(d), 1, after, before, group, group_options, group_time, options, NULL, NULL, @@ -1006,8 +1006,8 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { - if(!contexts || simple_pattern_matches(contexts, st->context)) - dictionary_set(charts, st->name, NULL, 0); + if(!contexts || simple_pattern_matches(contexts, rrdset_context(st))) + dictionary_set(charts, rrdset_name(st), NULL, 0); } } rrdhost_unlock(host); diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index a21c3dabff..6c295215b6 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -701,8 +701,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c if (context && !chart) { RRDSET *st1; - uint32_t context_hash = simple_hash(context); - SIMPLE_PATTERN *chart_label_key_pattern = NULL; if(chart_label_key) chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT); @@ -711,14 +709,16 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c if(chart_labels_filter) chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT); + STRING *context_string = string_strdupz(context); rrdhost_rdlock(host); rrdset_foreach_read(st1, host) { - if (st1->hash_context == context_hash && !strcmp(st1->context, context) && + if (st1->context == context_string && (!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_label_key_pattern, ':')) && (!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_labels_filter_pattern, ':'))) build_context_param_list(owa, &context_param_list, st1); } rrdhost_unlock(host); + string_freez(context_string); if (likely(context_param_list && context_param_list->rd)) // Just set the first one st = context_param_list->rd->rrdset; @@ -1055,7 +1055,7 @@ static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST int alarm_normal = 0, alarm_warn = 0, alarm_crit = 0; RRDCALC *rc; rrdhost_rdlock(host); - for(rc = host->alarms; rc ; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; @@ -1086,7 +1086,7 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) { if (count > 0) buffer_strcat(wb, ",\n"); - buffer_sprintf(wb, "\t\t\"%s\"", host->hostname); + buffer_sprintf(wb, "\t\t\"%s\"", rrdhost_hostname(host)); count++; } @@ -1101,7 +1101,7 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) { buffer_sprintf( wb, "\t\t{ \"guid\": \"%s\", \"hostname\": \"%s\", \"reachable\": %s, \"hops\": %d" , host->machine_guid - , host->hostname + , rrdhost_hostname(host) , (host->receiver || host == localhost) ? "true" : "false" , host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1 ); @@ -1148,7 +1148,7 @@ extern int aclk_connected; inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb) { buffer_strcat(wb, "{\n"); - buffer_sprintf(wb, "\t\"version\": \"%s\",\n", host->program_version); + buffer_sprintf(wb, "\t\"version\": \"%s\",\n", rrdhost_program_version(host)); buffer_sprintf(wb, "\t\"uid\": \"%s\",\n", host->machine_guid); web_client_api_request_v1_info_mirrored_hosts(wb); diff --git a/web/server/web_client.c b/web/server/web_client.c index d287cec032..9865ce671b 100644 --- a/web/server/web_client.c +++ b/web/server/web_client.c @@ -1305,11 +1305,9 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch if(url && *url) strncpyz(&w->last_url[1], url, NETDATA_WEB_REQUEST_URL_SIZE - 1); else w->last_url[1] = '\0'; - uint32_t hash = simple_hash(tok); - - host = rrdhost_find_by_hostname(tok, hash); + host = rrdhost_find_by_hostname(tok); if (!host) - host = rrdhost_find_by_guid(tok, hash); + host = rrdhost_find_by_guid(tok); if (!host) { host = sql_create_host_by_uuid(tok); if (likely(host)) {