opensourcemirror
/
netdata
mirror of https://github.com/netdata/netdata.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Deduplicate all netdata strings (#13570) 

* rrdfamily

* rrddim

* rrdset plugin and module names

* rrdset units

* rrdset type

* rrdset family

* rrdset title

* rrdset title more

* rrdset context

* rrdcalctemplate context and removal of context hash from rrdset

* strings statistics

* rrdset name

* rearranged members of rrdset

* eliminate rrdset name hash; rrdcalc chart converted to STRING

* rrdset id, eliminated rrdset hash

* rrdcalc, alarm_entry, alert_config and some of rrdcalctemplate

* rrdcalctemplate

* rrdvar

* eval_variable

* rrddimvar and rrdsetvar

* rrdhost hostname, os and tags

* fix master commits

* added thread cache; implemented string_dup without locks

* faster thread cache

* rrdset and rrddim now use dictionaries for indexing

* rrdhost now uses dictionary

* rrdfamily now uses DICTIONARY

* rrdvar using dictionary instead of AVL

* allocate the right size to rrdvar flag members

* rrdhost remaining char * members to STRING *

* better error handling on indexing

* strings now use a read/write lock to allow parallel searches to the index

* removed AVL support from dictionaries; implemented STRING with native Judy calls

* string releases should be negative

* only 31 bits are allowed for enum flags

* proper locking on strings

* string threading unittest and fixes

* fix lgtm finding

* fixed naming

* stream chart/dimension definitions at the beginning of a streaming session

* thread stack variable is undefined on thread cancel

* rrdcontext garbage collect per host on startup

* worker control in garbage collection

* relaxed deletion of rrdmetrics

* type checking on dictfe

* netdata chart to monitor rrdcontext triggers

* Group chart label updates

* rrdcontext better handling of collected rrdsets

* rrdpush incremental transmition of definitions should use as much buffer as possible

* require 1MB per chart

* empty the sender buffer before enabling metrics streaming

* fill up to 50% of buffer

* reset signaling metrics sending

* use the shared variable for status

* use separate host flag for enabling streaming of metrics

* make sure the flag is clear

* add logging for streaming

* add logging for streaming on buffer overflow

* circular_buffer proper sizing

* removed obsolete logs

* do not execute worker jobs if not necessary

* better messages about compression disabling

* proper use of flags and updating rrdset last access time every time the obsoletion flag is flipped

* monitor stream sender used buffer ratio

* Update exporting unit tests

* no need to compare label value with strcmp

* streaming send workers now monitor bandwidth

* workers now use strings

* streaming receiver monitors incoming bandwidth

* parser shift of worker ids

* minor fixes

* Group chart label updates

* Populate context with dimensions that have data

* Fix chart id

* better shift of parser worker ids

* fix for streaming compression

* properly count received bytes

* ensure LZ4 compression ring buffer does not wrap prematurely

* do not stream empty charts; do not process empty instances in rrdcontext

* need_to_send_chart_definition() does not need an rrdset lock any more

* rrdcontext objects are collected, after data have been written to the db

* better logging of RRDCONTEXT transitions

* always set all variables needed by the worker utilization charts

* implemented double linked list for most objects; eliminated alarm indexes from rrdhost; and many more fixes

* lockless strings design - string_dup() and string_freez() are totally lockless when they dont need to touch Judy - only Judy is protected with a read/write lock

* STRING code re-organization for clarity

* thread_cache improvements; double numbers precision on worker threads

* STRING_ENTRY now shadown STRING, so no duplicate definition is required; string_length() renamed to string_strlen() to follow the paradigm of all other functions, STRING internal statistics are now only compiled with NETDATA_INTERNAL_CHECKS

* rrdhost index by hostname now cleans up; aclk queries of archieved hosts do not index hosts

* Add index to speed up database context searches

* Removed last_updated optimization (was also buggy after latest merge with master)

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>
This commit is contained in:
Costa Tsaousis 2022-09-05 19:31:06 +03:00 committed by GitHub
parent 544aef1fde
commit 5e1b95cf92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
99 changed files with 4271 additions and 3468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -1498,6 +1498,7 @@ if(BUILD_TESTING)
exporting/tests/netdata_doubles.c
exporting/tests/system_doubles.c
database/rrdlabels.c
database/rrdvar.c
)
set(TEST_NAME exporting_engine)
set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS)

1
Makefile.am
View File

@ -1252,6 +1252,7 @@ if ENABLE_UNITTESTS
$(EXPORTING_ENGINE_FILES) \
$(LIBNETDATA_FILES) \
database/rrdlabels.c \
database/rrdvar.c \
$(NULL)
exporting_tests_exporting_engine_testdriver_CFLAGS = \
$(AM_CFLAGS) \

10
aclk/aclk.c
View File

@ -762,7 +762,7 @@ void aclk_host_state_update(RRDHOST *host, int cmd)
node_instance_creation_t node_instance_creation = {
.claim_id = localhost->aclk_state.claimed_id,
.hops = host->system_info->hops,
.hostname = host->hostname,
.hostname = rrdhost_hostname(host),
.machine_guid = host->machine_guid
};
create_query->data.bin_payload.payload = generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation);
@ -829,7 +829,7 @@ void aclk_send_node_instances()
char host_id[UUID_STR_LEN];
uuid_unparse_lower(list->host_id, host_id);
RRDHOST *host = rrdhost_find_by_guid(host_id, 0);
RRDHOST *host = rrdhost_find_by_guid(host_id);
struct capability caps[] = {
{ .name = "proto", .version = 1, .enabled = 1 },
{ .name = "ml", .version = ml_capable(localhost), .enabled = host ? ml_enabled(host) : 0 },
@ -867,7 +867,7 @@ void aclk_send_node_instances()
rrdhost_aclk_state_unlock(localhost);
info("Queuing registration for host=%s, hops=%d",(char*)node_instance_creation.machine_guid,
list->hops);
freez(node_instance_creation.machine_guid);
freez((void *)node_instance_creation.machine_guid);
aclk_queue_query(create_query);
}
freez(list->hostname);
@ -992,7 +992,7 @@ char *aclk_state(void)
RRDHOST *host;
rrd_rdlock();
rrdhost_foreach_read(host) {
buffer_sprintf(wb, "\n\n> Node Instance for mGUID: \"%s\" hostname \"%s\"\n", host->machine_guid, host->hostname);
buffer_sprintf(wb, "\n\n> Node Instance for mGUID: \"%s\" hostname \"%s\"\n", host->machine_guid, rrdhost_hostname(host));
buffer_strcat(wb, "\tClaimed ID: ");
rrdhost_aclk_state_lock(host);
@ -1179,7 +1179,7 @@ char *aclk_state_json(void)
rrdhost_foreach_read(host) {
json_object *nodeinstance = json_object_new_object();
tmp = json_object_new_string(host->hostname);
tmp = json_object_new_string(rrdhost_hostname(host));
json_object_object_add(nodeinstance, "hostname", tmp);
tmp = json_object_new_string(host->machine_guid);

6
aclk/aclk_query.c
View File

@ -38,9 +38,7 @@ static RRDHOST *node_id_2_rrdhost(const char *node_id)
int res;
uuid_t node_id_bin, host_id_bin;
rrd_rdlock();
RRDHOST *host = find_host_by_node_id((char *) node_id);
rrd_unlock();
RRDHOST *host = find_host_by_node_id((char *)node_id);
if (host)
return host;
@ -54,7 +52,7 @@ static RRDHOST *node_id_2_rrdhost(const char *node_id)
return NULL;
}
uuid_unparse_lower(host_id_bin, host_id);
return rrdhost_find_by_guid(host_id, 0);
return rrdhost_find_by_guid(host_id);
}
#define NODE_ID_QUERY "/node/"

4
aclk/aclk_rx_msgs.c
View File

@ -274,7 +274,7 @@ int create_node_instance_result(const char *msg, size_t msg_len)
.node_id = res.node_id
};
RRDHOST *host = rrdhost_find_by_guid(res.machine_guid, 0);
RRDHOST *host = rrdhost_find_by_guid(res.machine_guid);
if (host) {
// not all host must have RRDHOST struct created for them
// if they never connected during runtime of agent
@ -527,7 +527,7 @@ unsigned int aclk_init_rx_msg_handlers(void)
return i;
}
void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t msg_len, const char *topic)
void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t msg_len, const char *topic __maybe_unused)
{
if (aclk_stats_enabled) {
ACLK_STATS_LOCK;

6
aclk/schema-wrappers/node_creation.h
View File

@ -8,9 +8,9 @@ extern "C" {
#endif
typedef struct {
char* claim_id;
char* machine_guid;
char* hostname;
const char *claim_id;
const char *machine_guid;
const char *hostname;
int32_t hops;
} node_instance_creation_t;

52
aclk/schema-wrappers/node_info.h
View File

@ -19,41 +19,27 @@ struct machine_learning_info {
};
struct aclk_node_info {
char *name;
char *os;
char *os_name;
char *os_version;
char *kernel_name;
char *kernel_version;
char *architecture;
const char *name;
const char *os;
const char *os_name;
const char *os_version;
const char *kernel_name;
const char *kernel_version;
const char *architecture;
uint32_t cpus;
char *cpu_frequency;
char *memory;
char *disk_space;
char *version;
char *release_channel;
char *timezone;
char *virtualization_type;
char *container_type;
char *custom_info;
char *machine_guid;
const char *cpu_frequency;
const char *memory;
const char *disk_space;
const char *version;
const char *release_channel;
const char *timezone;
const char *virtualization_type;
const char *container_type;
const char *custom_info;
const char *machine_guid;
DICTIONARY *host_labels_ptr;
struct machine_learning_info ml_info;
};
@ -72,8 +58,8 @@ struct update_node_info {
};
struct collector_info {
char *module;
char *plugin;
const char *module;
const char *plugin;
};
struct update_node_collectors {

5
collectors/cgroups.plugin/tests/test_doubles.c
View File

@ -157,3 +157,8 @@ void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, cha
void rrdcalc_update_rrdlabels(RRDSET *st) {
(void)st;
}
void db_execute(const char *cmd)
{
UNUSED(cmd);
}

42
collectors/plugins.d/pluginsd_parser.c
View File

@ -105,13 +105,13 @@ PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *
if (rv)
rrdvar_custom_host_variable_set(host, rv, value);
else
error("cannot find/create HOST VARIABLE '%s' on host '%s'", name, host->hostname);
error("cannot find/create HOST VARIABLE '%s' on host '%s'", name, rrdhost_hostname(host));
} else {
RRDSETVAR *rs = rrdsetvar_custom_chart_variable_create(st, name);
if (rs)
rrdsetvar_custom_chart_variable_set(rs, value);
else
error("cannot find/create CHART VARIABLE '%s' on host '%s', chart '%s'", name, host->hostname, st->id);
error("cannot find/create CHART VARIABLE '%s' on host '%s', chart '%s'", name, rrdhost_hostname(host), rrdset_id(st));
}
return PARSER_RC_OK;
}
@ -184,7 +184,7 @@ PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, DICTIONARY *n
{
RRDSET *st = ((PARSER_USER_OBJECT *)user)->st;
if (unlikely(!st)) {
error("requested CLABEL_COMMIT on host '%s', without a BEGIN, ignoring it.", host->hostname);
error("requested CLABEL_COMMIT on host '%s', without a BEGIN, ignoring it.", rrdhost_hostname(host));
return PARSER_RC_OK;
}
@ -215,7 +215,7 @@ PARSER_RC pluginsd_set(char **words, void *user, PLUGINSD_ACTION *plugins_actio
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
if (unlikely(!dimension || !*dimension)) {
error("requested a SET on chart '%s' of host '%s', without a dimension. Disabling it.", st->id, host->hostname);
error("requested a SET on chart '%s' of host '%s', without a dimension. Disabling it.", rrdset_id(st), rrdhost_hostname(host));
goto disable;
}
@ -225,19 +225,19 @@ PARSER_RC pluginsd_set(char **words, void *user, PLUGINSD_ACTION *plugins_actio
if (unlikely(!st)) {
error(
"requested a SET on dimension %s with value %s on host '%s', without a BEGIN. Disabling it.", dimension,
value ? value : "<nothing>", host->hostname);
value ? value : "<nothing>", rrdhost_hostname(host));
goto disable;
}
if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
debug(D_PLUGINSD, "is setting dimension %s/%s to %s", st->id, dimension, value ? value : "<nothing>");
debug(D_PLUGINSD, "is setting dimension '%s'/'%s' to '%s'", rrdset_id(st), dimension, value ? value : "<nothing>");
if (value) {
RRDDIM *rd = rrddim_find(st, dimension);
if (unlikely(!rd)) {
error(
"requested a SET to dimension with id '%s' on stats '%s' (%s) on host '%s', which does not exist. Disabling it.",
dimension, st->name, st->id, st->rrdhost->hostname);
dimension, rrdset_name(st), rrdset_id(st), rrdhost_hostname(st->rrdhost));
goto disable;
} else {
if (plugins_action->set_action) {
@ -262,13 +262,13 @@ PARSER_RC pluginsd_begin(char **words, void *user, PLUGINSD_ACTION *plugins_act
RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host;
if (unlikely(!id)) {
error("requested a BEGIN without a chart id for host '%s'. Disabling it.", host->hostname);
error("requested a BEGIN without a chart id for host '%s'. Disabling it.", rrdhost_hostname(host));
goto disable;
}
st = rrdset_find(host, id);
if (unlikely(!st)) {
error("requested a BEGIN on chart '%s', which does not exist on host '%s'. Disabling it.", id, host->hostname);
error("requested a BEGIN on chart '%s', which does not exist on host '%s'. Disabling it.", id, rrdhost_hostname(host));
goto disable;
}
((PARSER_USER_OBJECT *)user)->st = st;
@ -294,13 +294,13 @@ PARSER_RC pluginsd_end(char **words, void *user, PLUGINSD_ACTION *plugins_actio
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
if (unlikely(!st)) {
error("requested an END, without a BEGIN on host '%s'. Disabling it.", host->hostname);
error("requested an END, without a BEGIN on host '%s'. Disabling it.", rrdhost_hostname(host));
((PARSER_USER_OBJECT *) user)->enabled = 0;
return PARSER_RC_ERROR;
}
if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
debug(D_PLUGINSD, "requested an END on chart %s", st->id);
debug(D_PLUGINSD, "requested an END on chart '%s'", rrdset_id(st));
((PARSER_USER_OBJECT *) user)->st = NULL;
((PARSER_USER_OBJECT *) user)->count++;
@ -343,7 +343,7 @@ PARSER_RC pluginsd_chart(char **words, void *user, PLUGINSD_ACTION *plugins_act
// make sure we have the required variables
if (unlikely((!type || !*type || !id || !*id))) {
if (likely(host))
error("requested a CHART, without a type.id, on host '%s'. Disabling it.", host->hostname);
error("requested a CHART, without a type.id, on host '%s'. Disabling it.", rrdhost_hostname(host));
else
error("requested a CHART, without a type.id. Disabling it.");
((PARSER_USER_OBJECT *) user)->enabled = 0;
@ -424,13 +424,13 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION *plugins
if (unlikely(!id)) {
error(
"requested a DIMENSION, without an id, host '%s' and chart '%s'. Disabling it.", host->hostname,
st ? st->id : "UNSET");
"requested a DIMENSION, without an id, host '%s' and chart '%s'. Disabling it.", rrdhost_hostname(host),
st ? rrdset_id(st) : "UNSET");
goto disable;
}
if (unlikely(!st && !((PARSER_USER_OBJECT *) user)->st_exists)) {
error("requested a DIMENSION, without a CHART, on host '%s'. Disabling it.", host->hostname);
error("requested a DIMENSION, without a CHART, on host '%s'. Disabling it.", rrdhost_hostname(host));
goto disable;
}
@ -455,7 +455,7 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION *plugins
debug(
D_PLUGINSD,
"creating dimension in chart %s, id='%s', name='%s', algorithm='%s', multiplier=%ld, divisor=%ld, hidden='%s'",
st->id, id, name ? name : "", rrd_algorithm_name(rrd_algorithm_id(algorithm)), multiplier, divisor,
rrdset_id(st), id, name ? name : "", rrd_algorithm_name(rrd_algorithm_id(algorithm)), multiplier, divisor,
options ? options : "");
if (plugins_action->dimension_action) {
@ -494,7 +494,7 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_
}
if (unlikely(!name || !*name)) {
error("requested a VARIABLE on host '%s', without a variable name. Disabling it.", host->hostname);
error("requested a VARIABLE on host '%s', without a variable name. Disabling it.", rrdhost_hostname(host));
((PARSER_USER_OBJECT *)user)->enabled = 0;
return PARSER_RC_ERROR;
}
@ -504,12 +504,12 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_
if (unlikely(!value)) {
error("cannot set %s VARIABLE '%s' on host '%s' to an empty value", (global) ? "HOST" : "CHART", name,
host->hostname);
rrdhost_hostname(host));
return PARSER_RC_OK;
}
if (!global && !st) {
error("cannot find/create CHART VARIABLE '%s' on host '%s' without a chart", name, host->hostname);
error("cannot find/create CHART VARIABLE '%s' on host '%s' without a chart", name, rrdhost_hostname(host));
return PARSER_RC_OK;
}
@ -519,10 +519,10 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_
if (endptr == value)
error(
"the value '%s' of VARIABLE '%s' on host '%s' cannot be parsed as a number", value, name,
host->hostname);
rrdhost_hostname(host));
else
error(
"the value '%s' of VARIABLE '%s' on host '%s' has leftovers: '%s'", value, name, host->hostname,
"the value '%s' of VARIABLE '%s' on host '%s' has leftovers: '%s'", value, name, rrdhost_hostname(host),
endptr);
}

4
collectors/proc.plugin/ipc.c
View File

@ -499,11 +499,11 @@ int do_ipc(int update_every, usec_t dt) {
do_msg = CONFIG_BOOLEAN_NO;
}
else if(unlikely(!message_queue_root)) {
info("Making chart %s (%s) obsolete since it does not have any dimensions", st_msq_messages->name, st_msq_messages->id);
info("Making chart %s (%s) obsolete since it does not have any dimensions", rrdset_name(st_msq_messages), rrdset_id(st_msq_messages));
rrdset_is_obsolete(st_msq_messages);
st_msq_messages = NULL;
info("Making chart %s (%s) obsolete since it does not have any dimensions", st_msq_bytes->name, st_msq_bytes->id);
info("Making chart %s (%s) obsolete since it does not have any dimensions", rrdset_name(st_msq_bytes), rrdset_id(st_msq_bytes));
rrdset_is_obsolete(st_msq_bytes);
st_msq_bytes = NULL;
}

2
collectors/proc.plugin/proc_interrupts.c
View File

@ -173,7 +173,7 @@ int do_proc_interrupts(int update_every, usec_t dt) {
// some interrupt may have changed without changing the total number of lines
// if the same number of interrupts have been added and removed between two
// calls of this function.
if(unlikely(!irr->rd || strncmp(irr->rd->name, irr->name, MAX_INTERRUPT_NAME) != 0)) {
if(unlikely(!irr->rd || strncmp(rrddim_name(irr->rd), irr->name, MAX_INTERRUPT_NAME) != 0)) {
irr->rd = rrddim_add(st_system_interrupts, irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rrddim_set_name(st_system_interrupts, irr->rd, irr->name);

2
collectors/proc.plugin/proc_softirqs.c
View File

@ -153,7 +153,7 @@ int do_proc_softirqs(int update_every, usec_t dt) {
// some interrupt may have changed without changing the total number of lines
// if the same number of interrupts have been added and removed between two
// calls of this function.
if(unlikely(!irr->rd || strncmp(irr->name, irr->rd->name, MAX_INTERRUPT_NAME) != 0)) {
if(unlikely(!irr->rd || strncmp(irr->name, rrddim_name(irr->rd), MAX_INTERRUPT_NAME) != 0)) {
irr->rd = rrddim_add(st_system_softirqs, irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rrddim_set_name(st_system_softirqs, irr->rd, irr->name);

2
collectors/statsd.plugin/statsd.c
View File

@ -2228,7 +2228,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
statsd_add_dim_to_app_chart(app, chart, dim);
if (unlikely(dim->value_ptr)) {
debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr);
debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, rrddim_id(dim->rd), chart->id, rrdset_id(chart->st), app->name, *dim->value_ptr);
rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr);
}
}

14
daemon/analytics.c
View File

@ -10,8 +10,8 @@ extern void analytics_build_info (BUFFER *b);
extern int aclk_connected;
struct collector {
char *plugin;
char *module;
const char *plugin;
const char *module;
};
struct array_printer {
@ -286,8 +286,10 @@ void analytics_collectors(void)
rrdset_foreach_read(st, localhost)
{
if (rrdset_is_available_for_viewers(st)) {
struct collector col = { .plugin = st->plugin_name ? st->plugin_name : "",
.module = st->module_name ? st->module_name : "" };
struct collector col = {
.plugin = rrdset_plugin_name(st),
.module = rrdset_module_name(st)
};
snprintfz(name, 499, "%s:%s", col.plugin, col.module);
dictionary_set(dict, name, &col, sizeof(struct collector));
}
@ -441,7 +443,7 @@ void analytics_alarms(void)
int alarm_warn = 0, alarm_crit = 0, alarm_normal = 0;
char b[10];
RRDCALC *rc;
for (rc = localhost->alarms; rc; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(localhost, rc) {
if (unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
continue;
@ -539,7 +541,7 @@ void analytics_gather_mutable_meta_data(void)
analytics_alarms_notifications();
analytics_set_data(
&analytics_data.netdata_config_is_parent, (localhost->next || configured_as_parent()) ? "true" : "false");
&analytics_data.netdata_config_is_parent, (rrdhost_hosts_available() > 1 || configured_as_parent()) ? "true" : "false");
char *claim_id = get_agent_claimid();
analytics_set_data(&analytics_data.netdata_host_agent_claimed, claim_id ? "true" : "false");

383
daemon/global_statistics.c
View File

@ -9,8 +9,9 @@
#define WORKER_JOB_WORKERS 2
#define WORKER_JOB_DBENGINE 3
#define WORKER_JOB_HEARTBEAT 4
#define WORKER_JOB_STRINGS 5
#if WORKER_UTILIZATION_MAX_JOB_TYPES < 5
#if WORKER_UTILIZATION_MAX_JOB_TYPES < 6
#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5
#endif
@ -38,6 +39,10 @@ static struct global_statistics {
volatile uint64_t sqlite3_queries_failed_locked;
volatile uint64_t sqlite3_rows;
volatile uint64_t rrdcontext_metric_triggers;
volatile uint64_t rrdcontext_instance_triggers;
volatile uint64_t rrdcontext_context_triggers;
} global_statistics = {
.connected_clients = 0,
.web_requests = 0,
@ -80,6 +85,18 @@ void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_genera
__atomic_fetch_add(&global_statistics.rrdr_result_points_generated, result_points_generated, __ATOMIC_RELAXED);
}
void rrdcontext_triggered_update_on_rrdmetric() {
__atomic_fetch_add(&global_statistics.rrdcontext_metric_triggers, 1, __ATOMIC_RELAXED);
}
void rrdcontext_triggered_update_on_rrdinstance() {
__atomic_fetch_add(&global_statistics.rrdcontext_instance_triggers, 1, __ATOMIC_RELAXED);
}
void rrdcontext_triggered_update_on_rrdcontext() {
__atomic_fetch_add(&global_statistics.rrdcontext_context_triggers, 1, __ATOMIC_RELAXED);
}
void finished_web_request_statistics(uint64_t dt,
uint64_t bytes_received,
uint64_t bytes_sent,
@ -133,6 +150,10 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
gs->sqlite3_queries_failed_busy = __atomic_load_n(&global_statistics.sqlite3_queries_failed_busy, __ATOMIC_RELAXED);
gs->sqlite3_queries_failed_locked = __atomic_load_n(&global_statistics.sqlite3_queries_failed_locked, __ATOMIC_RELAXED);
gs->sqlite3_rows = __atomic_load_n(&global_statistics.sqlite3_rows, __ATOMIC_RELAXED);
gs->rrdcontext_metric_triggers = __atomic_load_n(&global_statistics.rrdcontext_metric_triggers, __ATOMIC_RELAXED);
gs->rrdcontext_instance_triggers = __atomic_load_n(&global_statistics.rrdcontext_instance_triggers, __ATOMIC_RELAXED);
gs->rrdcontext_context_triggers = __atomic_load_n(&global_statistics.rrdcontext_context_triggers, __ATOMIC_RELAXED);
}
static void global_statistics_charts(void) {
@ -579,6 +600,42 @@ static void global_statistics_charts(void) {
rrdset_done(st_sqlite3_rows);
}
// ----------------------------------------------------------------
if(gs.rrdcontext_context_triggers || gs.rrdcontext_instance_triggers || gs.rrdcontext_metric_triggers) {
static RRDSET *st_rrdcontext_triggers = NULL;
static RRDDIM *rd_context = NULL, *rd_instance = NULL, *rd_metric = NULL;
if (unlikely(!st_rrdcontext_triggers)) {
st_rrdcontext_triggers = rrdset_create_localhost(
"netdata"
, "rrdcontext_triggers"
, NULL
, "rrdcontext"
, NULL
, "Netdata RRD context triggers"
, "triggers/s"
, "netdata"
, "stats"
, 131200
, localhost->rrd_update_every
, RRDSET_TYPE_LINE
);
rd_metric = rrddim_add(st_rrdcontext_triggers, "metric", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_instance = rrddim_add(st_rrdcontext_triggers, "instance", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_context = rrddim_add(st_rrdcontext_triggers, "context", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
}
else
rrdset_next(st_rrdcontext_triggers);
rrddim_set_by_pointer(st_rrdcontext_triggers, rd_metric, (collected_number)gs.rrdcontext_metric_triggers);
rrddim_set_by_pointer(st_rrdcontext_triggers, rd_instance, (collected_number)gs.rrdcontext_instance_triggers);
rrddim_set_by_pointer(st_rrdcontext_triggers, rd_context, (collected_number)gs.rrdcontext_context_triggers);
rrdset_done(st_rrdcontext_triggers);
}
}
static void dbengine_statistics_charts(void) {
@ -988,6 +1045,93 @@ static void dbengine_statistics_charts(void) {
#endif
}
static void update_strings_charts() {
static RRDSET *st_ops = NULL, *st_entries = NULL, *st_mem = NULL;
static RRDDIM *rd_ops_inserts = NULL, *rd_ops_deletes = NULL, *rd_ops_searches = NULL, *rd_ops_duplications = NULL, *rd_ops_releases = NULL;
static RRDDIM *rd_entries_entries = NULL, *rd_entries_refs = NULL;
static RRDDIM *rd_mem = NULL;
size_t inserts, deletes, searches, entries, references, memory, duplications, releases;
string_statistics(&inserts, &deletes, &searches, &entries, &references, &memory, &duplications, &releases);
if (unlikely(!st_ops)) {
st_ops = rrdset_create_localhost(
"netdata"
, "strings_ops"
, NULL
, "strings"
, NULL
, "Strings operations"
, "ops/s"
, "netdata"
, "stats"
, 910000
, localhost->rrd_update_every
, RRDSET_TYPE_LINE);
rd_ops_inserts = rrddim_add(st_ops, "inserts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_ops_deletes = rrddim_add(st_ops, "deletes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_ops_searches = rrddim_add(st_ops, "searches", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_ops_duplications = rrddim_add(st_ops, "duplications", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_ops_releases = rrddim_add(st_ops, "releases", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
} else
rrdset_next(st_ops);
rrddim_set_by_pointer(st_ops, rd_ops_inserts, (collected_number)inserts);
rrddim_set_by_pointer(st_ops, rd_ops_deletes, (collected_number)deletes);
rrddim_set_by_pointer(st_ops, rd_ops_searches, (collected_number)searches);
rrddim_set_by_pointer(st_ops, rd_ops_duplications, (collected_number)duplications);
rrddim_set_by_pointer(st_ops, rd_ops_releases, (collected_number)releases);
rrdset_done(st_ops);
if (unlikely(!st_entries)) {
st_entries = rrdset_create_localhost(
"netdata"
, "strings_entries"
, NULL
, "strings"
, NULL
, "Strings entries"
, "entries"
, "netdata"
, "stats"
, 910001
, localhost->rrd_update_every
, RRDSET_TYPE_AREA);
rd_entries_entries = rrddim_add(st_entries, "entries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rd_entries_refs = rrddim_add(st_entries, "references", NULL, 1, -1, RRD_ALGORITHM_ABSOLUTE);
} else
rrdset_next(st_entries);
rrddim_set_by_pointer(st_entries, rd_entries_entries, (collected_number)entries);
rrddim_set_by_pointer(st_entries, rd_entries_refs, (collected_number)references);
rrdset_done(st_entries);
if (unlikely(!st_mem)) {
st_mem = rrdset_create_localhost(
"netdata"
, "strings_memory"
, NULL
, "strings"
, NULL
, "Strings memory"
, "bytes"
, "netdata"
, "stats"
, 910001
, localhost->rrd_update_every
, RRDSET_TYPE_AREA);
rd_mem = rrddim_add(st_mem, "memory", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
} else
rrdset_next(st_mem);
rrddim_set_by_pointer(st_mem, rd_mem, (collected_number)memory);
rrdset_done(st_mem);
}
static void update_heartbeat_charts() {
static RRDSET *st_heartbeat = NULL;
static RRDDIM *rd_heartbeat_min = NULL;
@ -1032,13 +1176,26 @@ static void update_heartbeat_charts() {
#define WORKERS_MIN_PERCENT_DEFAULT 10000.0
struct worker_job_type {
char name[WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH + 1];
struct worker_job_type_gs {
STRING *name;
STRING *units;
size_t jobs_started;
usec_t busy_time;
RRDDIM *rd_jobs_started;
RRDDIM *rd_busy_time;
WORKER_METRIC_TYPE type;
NETDATA_DOUBLE min_value;
NETDATA_DOUBLE max_value;
NETDATA_DOUBLE sum_value;
size_t count_value;
RRDSET *st;
RRDDIM *rd_min;
RRDDIM *rd_max;
RRDDIM *rd_avg;
};
struct worker_thread {
@ -1071,7 +1228,7 @@ struct worker_utilization {
char *name_lowercase;
struct worker_job_type per_job_type[WORKER_UTILIZATION_MAX_JOB_TYPES];
struct worker_job_type_gs per_job_type[WORKER_UTILIZATION_MAX_JOB_TYPES];
size_t workers_registered;
size_t workers_busy;
@ -1170,14 +1327,16 @@ static void workers_total_cpu_utilization_chart(void) {
if(!wu->workers_cpu_registered) continue;
if(!wu->rd_total_cpu_utilizaton)
wu->rd_total_cpu_utilizaton = rrddim_add(st, wu->name_lowercase, NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE);
wu->rd_total_cpu_utilizaton = rrddim_add(st, wu->name_lowercase, NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(st, wu->rd_total_cpu_utilizaton, (collected_number)((double)wu->workers_cpu_total * 10000.0));
rrddim_set_by_pointer(st, wu->rd_total_cpu_utilizaton, (collected_number)((double)wu->workers_cpu_total * 100.0));
}
rrdset_done(st);
}
#define WORKER_CHART_DECIMAL_PRECISION 100
static void workers_utilization_update_chart(struct worker_utilization *wu) {
if(!wu->workers_registered) return;
@ -1215,28 +1374,28 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
// we add the min and max dimensions only when we have multiple workers
if(unlikely(!wu->rd_workers_time_min && wu->workers_registered > 1))
wu->rd_workers_time_min = rrddim_add(wu->st_workers_time, "min", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
wu->rd_workers_time_min = rrddim_add(wu->st_workers_time, "min", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
if(unlikely(!wu->rd_workers_time_max && wu->workers_registered > 1))
wu->rd_workers_time_max = rrddim_add(wu->st_workers_time, "max", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
wu->rd_workers_time_max = rrddim_add(wu->st_workers_time, "max", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
if(unlikely(!wu->rd_workers_time_avg))
wu->rd_workers_time_avg = rrddim_add(wu->st_workers_time, "average", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
wu->rd_workers_time_avg = rrddim_add(wu->st_workers_time, "average", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
rrdset_next(wu->st_workers_time);
if(unlikely(wu->workers_min_busy_time == WORKERS_MIN_PERCENT_DEFAULT)) wu->workers_min_busy_time = 0.0;
if(wu->rd_workers_time_min)
rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_min, (collected_number)((double)wu->workers_min_busy_time * 10000.0));
rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_min, (collected_number)((double)wu->workers_min_busy_time * WORKER_CHART_DECIMAL_PRECISION));
if(wu->rd_workers_time_max)
rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_max, (collected_number)((double)wu->workers_max_busy_time * 10000.0));
rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_max, (collected_number)((double)wu->workers_max_busy_time * WORKER_CHART_DECIMAL_PRECISION));
if(wu->workers_total_duration == 0)
rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, 0);
else
rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, (collected_number)((double)wu->workers_total_busy_time * 100.0 * 10000.0 / (double)wu->workers_total_duration));
rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, (collected_number)((double)wu->workers_total_busy_time * 100.0 * WORKER_CHART_DECIMAL_PRECISION / (double)wu->workers_total_duration));
rrdset_done(wu->st_workers_time);
@ -1268,28 +1427,28 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
}
if (unlikely(!wu->rd_workers_cpu_min && wu->workers_registered > 1))
wu->rd_workers_cpu_min = rrddim_add(wu->st_workers_cpu, "min", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE);
wu->rd_workers_cpu_min = rrddim_add(wu->st_workers_cpu, "min", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
if (unlikely(!wu->rd_workers_cpu_max && wu->workers_registered > 1))
wu->rd_workers_cpu_max = rrddim_add(wu->st_workers_cpu, "max", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE);
wu->rd_workers_cpu_max = rrddim_add(wu->st_workers_cpu, "max", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
if(unlikely(!wu->rd_workers_cpu_avg))
wu->rd_workers_cpu_avg = rrddim_add(wu->st_workers_cpu, "average", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE);
wu->rd_workers_cpu_avg = rrddim_add(wu->st_workers_cpu, "average", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
rrdset_next(wu->st_workers_cpu);
if(unlikely(wu->workers_cpu_min == WORKERS_MIN_PERCENT_DEFAULT)) wu->workers_cpu_min = 0.0;
if(wu->rd_workers_cpu_min)
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_min, (collected_number)(wu->workers_cpu_min * 10000ULL));
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_min, (collected_number)(wu->workers_cpu_min * WORKER_CHART_DECIMAL_PRECISION));
if(wu->rd_workers_cpu_max)
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_max, (collected_number)(wu->workers_cpu_max * 10000ULL));
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_max, (collected_number)(wu->workers_cpu_max * WORKER_CHART_DECIMAL_PRECISION));
if(wu->workers_cpu_registered == 0)
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, 0);
else
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (NETDATA_DOUBLE)wu->workers_cpu_registered ));
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * WORKER_CHART_DECIMAL_PRECISION / (NETDATA_DOUBLE)wu->workers_cpu_registered ));
rrdset_done(wu->st_workers_cpu);
}
@ -1325,10 +1484,13 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
{
size_t i;
for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
if (wu->per_job_type[i].name[0]) {
if(unlikely(wu->per_job_type[i].type != WORKER_METRIC_IDLE_BUSY))
continue;
if (wu->per_job_type[i].name) {
if(unlikely(!wu->per_job_type[i].rd_jobs_started))
wu->per_job_type[i].rd_jobs_started = rrddim_add(wu->st_workers_jobs_per_job_type, wu->per_job_type[i].name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
wu->per_job_type[i].rd_jobs_started = rrddim_add(wu->st_workers_jobs_per_job_type, string2str(wu->per_job_type[i].name), NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(wu->st_workers_jobs_per_job_type, wu->per_job_type[i].rd_jobs_started, (collected_number)(wu->per_job_type[i].jobs_started));
}
@ -1367,10 +1529,13 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
{
size_t i;
for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
if (wu->per_job_type[i].name[0]) {
if(unlikely(wu->per_job_type[i].type != WORKER_METRIC_IDLE_BUSY))
continue;
if (wu->per_job_type[i].name) {
if(unlikely(!wu->per_job_type[i].rd_busy_time))
wu->per_job_type[i].rd_busy_time = rrddim_add(wu->st_workers_busy_per_job_type, wu->per_job_type[i].name, NULL, 1, USEC_PER_MS, RRD_ALGORITHM_ABSOLUTE);
wu->per_job_type[i].rd_busy_time = rrddim_add(wu->st_workers_busy_per_job_type, string2str(wu->per_job_type[i].name), NULL, 1, USEC_PER_MS, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(wu->st_workers_busy_per_job_type, wu->per_job_type[i].rd_busy_time, (collected_number)(wu->per_job_type[i].busy_time));
}
@ -1414,6 +1579,122 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
rrddim_set_by_pointer(wu->st_workers_threads, wu->rd_workers_threads_busy, (collected_number)(wu->workers_busy));
rrdset_done(wu->st_workers_threads);
}
// ----------------------------------------------------------------------
// custom metric types WORKER_METRIC_ABSOLUTE
{
size_t i;
for (i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES; i++) {
if(wu->per_job_type[i].type != WORKER_METRIC_ABSOLUTE)
continue;
if(!wu->per_job_type[i].count_value)
continue;
if(!wu->per_job_type[i].st) {
size_t job_name_len = string_strlen(wu->per_job_type[i].name);
if(job_name_len > RRD_ID_LENGTH_MAX) job_name_len = RRD_ID_LENGTH_MAX;
char job_name_sanitized[job_name_len + 1];
rrdset_strncpyz_name(job_name_sanitized, string2str(wu->per_job_type[i].name), job_name_len);
char name[RRD_ID_LENGTH_MAX + 1];
snprintfz(name, RRD_ID_LENGTH_MAX, "workers_%s_value_%s", wu->name_lowercase, job_name_sanitized);
char context[RRD_ID_LENGTH_MAX + 1];
snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.value.%s", wu->name_lowercase, job_name_sanitized);
char title[1000 + 1];
snprintf(title, 1000, "Netdata Workers %s Value of %s", wu->name_lowercase, string2str(wu->per_job_type[i].name));
wu->per_job_type[i].st = rrdset_create_localhost(
"netdata"
, name
, NULL
, wu->family
, context
, title
, (wu->per_job_type[i].units)?string2str(wu->per_job_type[i].units):"value"
, "netdata"
, "stats"
, wu->priority + 5
, localhost->rrd_update_every
, RRDSET_TYPE_LINE
);
wu->per_job_type[i].rd_min = rrddim_add(wu->per_job_type[i].st, "min", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
wu->per_job_type[i].rd_max = rrddim_add(wu->per_job_type[i].st, "max", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
wu->per_job_type[i].rd_avg = rrddim_add(wu->per_job_type[i].st, "average", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
}
else
rrdset_next(wu->per_job_type[i].st);
rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_min, (collected_number)(wu->per_job_type[i].min_value * WORKER_CHART_DECIMAL_PRECISION));
rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_max, (collected_number)(wu->per_job_type[i].max_value * WORKER_CHART_DECIMAL_PRECISION));
rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_avg, (collected_number)(wu->per_job_type[i].sum_value / wu->per_job_type[i].count_value * WORKER_CHART_DECIMAL_PRECISION));
rrdset_done(wu->per_job_type[i].st);
}
}
// ----------------------------------------------------------------------
// custom metric types WORKER_METRIC_INCREMENTAL
{
size_t i;
for (i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES; i++) {
if(wu->per_job_type[i].type != WORKER_METRIC_INCREMENTAL)
continue;
if(!wu->per_job_type[i].count_value)
continue;
if(!wu->per_job_type[i].st) {
size_t job_name_len = string_strlen(wu->per_job_type[i].name);
if(job_name_len > RRD_ID_LENGTH_MAX) job_name_len = RRD_ID_LENGTH_MAX;
char job_name_sanitized[job_name_len + 1];
rrdset_strncpyz_name(job_name_sanitized, string2str(wu->per_job_type[i].name), job_name_len);
char name[RRD_ID_LENGTH_MAX + 1];
snprintfz(name, RRD_ID_LENGTH_MAX, "workers_%s_rate_%s", wu->name_lowercase, job_name_sanitized);
char context[RRD_ID_LENGTH_MAX + 1];
snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.rate.%s", wu->name_lowercase, job_name_sanitized);
char title[1000 + 1];
snprintf(title, 1000, "Netdata Workers %s Rate of %s", wu->name_lowercase, string2str(wu->per_job_type[i].name));
wu->per_job_type[i].st = rrdset_create_localhost(
"netdata"
, name
, NULL
, wu->family
, context
, title
, (wu->per_job_type[i].units)?string2str(wu->per_job_type[i].units):"rate"
, "netdata"
, "stats"
, wu->priority + 5
, localhost->rrd_update_every
, RRDSET_TYPE_LINE
);
wu->per_job_type[i].rd_min = rrddim_add(wu->per_job_type[i].st, "min", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
wu->per_job_type[i].rd_max = rrddim_add(wu->per_job_type[i].st, "max", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
wu->per_job_type[i].rd_avg = rrddim_add(wu->per_job_type[i].st, "average", NULL, 1, WORKER_CHART_DECIMAL_PRECISION, RRD_ALGORITHM_ABSOLUTE);
}
else
rrdset_next(wu->per_job_type[i].st);
rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_min, (collected_number)(wu->per_job_type[i].min_value * WORKER_CHART_DECIMAL_PRECISION));
rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_max, (collected_number)(wu->per_job_type[i].max_value * WORKER_CHART_DECIMAL_PRECISION));
rrddim_set_by_pointer(wu->per_job_type[i].st, wu->per_job_type[i].rd_avg, (collected_number)(wu->per_job_type[i].sum_value / wu->per_job_type[i].count_value * WORKER_CHART_DECIMAL_PRECISION));
rrdset_done(wu->per_job_type[i].st);
}
}
}
static void workers_utilization_reset_statistics(struct worker_utilization *wu) {
@ -1440,6 +1721,11 @@ static void workers_utilization_reset_statistics(struct worker_utilization *wu)
wu->per_job_type[i].jobs_started = 0;
wu->per_job_type[i].busy_time = 0;
wu->per_job_type[i].min_value = NAN;
wu->per_job_type[i].max_value = NAN;
wu->per_job_type[i].sum_value = NAN;
wu->per_job_type[i].count_value = 0;
}
struct worker_thread *wt;
@ -1522,7 +1808,20 @@ static struct worker_thread *worker_thread_find_or_create(struct worker_utilizat
return wt;
}
static void worker_utilization_charts_callback(void *ptr, pid_t pid __maybe_unused, const char *thread_tag __maybe_unused, size_t utilization_usec __maybe_unused, size_t duration_usec __maybe_unused, size_t jobs_started __maybe_unused, size_t is_running __maybe_unused, const char **job_types_names __maybe_unused, size_t *job_types_jobs_started __maybe_unused, usec_t *job_types_busy_time __maybe_unused) {
static void worker_utilization_charts_callback(void *ptr
, pid_t pid __maybe_unused
, const char *thread_tag __maybe_unused
, size_t utilization_usec __maybe_unused
, size_t duration_usec __maybe_unused
, size_t jobs_started __maybe_unused
, size_t is_running __maybe_unused
, STRING **job_types_names __maybe_unused
, STRING **job_types_units __maybe_unused
, WORKER_METRIC_TYPE *job_types_metric_types __maybe_unused
, size_t *job_types_jobs_started __maybe_unused
, usec_t *job_types_busy_time __maybe_unused
, NETDATA_DOUBLE *job_types_custom_metrics __maybe_unused
) {
struct worker_utilization *wu = (struct worker_utilization *)ptr;
// find the worker_thread in the list
@ -1552,12 +1851,32 @@ static void worker_utilization_charts_callback(void *ptr, pid_t pid __maybe_unus
// accumulate per job type statistics
size_t i;
for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
if(!wu->per_job_type[i].name && job_types_names[i])
wu->per_job_type[i].name = string_dup(job_types_names[i]);
if(!wu->per_job_type[i].units && job_types_units[i])
wu->per_job_type[i].units = string_dup(job_types_units[i]);
wu->per_job_type[i].type = job_types_metric_types[i];
wu->per_job_type[i].jobs_started += job_types_jobs_started[i];
wu->per_job_type[i].busy_time += job_types_busy_time[i];
// new job type found
if(unlikely(!wu->per_job_type[i].name[0] && job_types_names[i]))
strncpyz(wu->per_job_type[i].name, job_types_names[i], WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH);
NETDATA_DOUBLE value = job_types_custom_metrics[i];
if(netdata_double_isnumber(value)) {
if(!wu->per_job_type[i].count_value) {
wu->per_job_type[i].count_value = 1;
wu->per_job_type[i].min_value = value;
wu->per_job_type[i].max_value = value;
wu->per_job_type[i].sum_value = value;
}
else {
wu->per_job_type[i].count_value++;
wu->per_job_type[i].sum_value += value;
if(value < wu->per_job_type[i].min_value) wu->per_job_type[i].min_value = value;
if(value > wu->per_job_type[i].max_value) wu->per_job_type[i].max_value = value;
}
}
}
// find its CPU utilization
@ -1607,6 +1926,14 @@ static void worker_utilization_finish(void) {
wu->name_lowercase = NULL;
}
for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
string_freez(wu->per_job_type[i].name);
wu->per_job_type[i].name = NULL;
string_freez(wu->per_job_type[i].units);
wu->per_job_type[i].units = NULL;
}
// mark all threads as not enabled
struct worker_thread *t;
for(t = wu->threads; t ; t = t->next) t->enabled = 0;
@ -1639,6 +1966,7 @@ void *global_statistics_main(void *ptr)
worker_register_job_name(WORKER_JOB_REGISTRY, "registry");
worker_register_job_name(WORKER_JOB_WORKERS, "workers");
worker_register_job_name(WORKER_JOB_DBENGINE, "dbengine");
worker_register_job_name(WORKER_JOB_STRINGS, "strings");
netdata_thread_cleanup_push(global_statistics_cleanup, ptr);
@ -1673,6 +2001,9 @@ void *global_statistics_main(void *ptr)
worker_is_busy(WORKER_JOB_HEARTBEAT);
update_heartbeat_charts();
worker_is_busy(WORKER_JOB_STRINGS);
update_strings_charts();
}
netdata_thread_cleanup_pop(1);

4
daemon/global_statistics.h
View File

@ -12,6 +12,10 @@ extern void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points
extern void sqlite3_query_completed(bool success, bool busy, bool locked);
extern void sqlite3_row_completed(void);
extern void rrdcontext_triggered_update_on_rrdmetric(void);
extern void rrdcontext_triggered_update_on_rrdinstance(void);
extern void rrdcontext_triggered_update_on_rrdcontext(void);
extern void finished_web_request_statistics(uint64_t dt,
uint64_t bytes_received,
uint64_t bytes_sent,

62
daemon/unit_test.c
View File

@ -1197,12 +1197,12 @@ int run_test(struct test *test)
fprintf(stderr, " > %s: feeding position %lu\n", test->name, c+1);
}
fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rd->name, test->feed[c].value);
fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rrddim_name(rd), test->feed[c].value);
rrddim_set(st, "dim1", test->feed[c].value);
last = test->feed[c].value;
if(rd2) {
fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rd2->name, test->feed2[c]);
fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rrddim_name(rd2), test->feed2[c]);
rrddim_set(st, "dim2", test->feed2[c]);
}
@ -1231,7 +1231,7 @@ int run_test(struct test *test)
int same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", %s\n",
test->name, rd->name, c+1,
test->name, rrddim_name(rd), c+1,
(int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start),
n, v, (same)?"OK":"### E R R O R ###");
@ -1243,7 +1243,7 @@ int run_test(struct test *test)
same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", %s\n",
test->name, rd2->name, c+1,
test->name, rrddim_name(rd2), c+1,
(int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start),
n, v, (same)?"OK":"### E R R O R ###");
if(!same) errors++;
@ -1258,39 +1258,39 @@ static int test_variable_renames(void) {
fprintf(stderr, "Creating chart\n");
RRDSET *st = rrdset_create_localhost("chart", "ID", NULL, "family", "context", "Unit Testing", "a value", "unittest", NULL, 1, 1, RRDSET_TYPE_LINE);
fprintf(stderr, "Created chart with id '%s', name '%s'\n", st->id, st->name);
fprintf(stderr, "Created chart with id '%s', name '%s'\n", rrdset_id(st), rrdset_name(st));
fprintf(stderr, "Creating dimension DIM1\n");
RRDDIM *rd1 = rrddim_add(st, "DIM1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rd1->id, rd1->name);
fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rrddim_id(rd1), rrddim_name(rd1));
fprintf(stderr, "Creating dimension DIM2\n");
RRDDIM *rd2 = rrddim_add(st, "DIM2", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rd2->id, rd2->name);
fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rrddim_id(rd2), rrddim_name(rd2));
fprintf(stderr, "Renaming chart to CHARTNAME1\n");
rrdset_set_name(st, "CHARTNAME1");
fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", st->id, st->name);
fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", rrdset_id(st), rrdset_name(st));
fprintf(stderr, "Renaming chart to CHARTNAME2\n");
rrdset_set_name(st, "CHARTNAME2");
fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", st->id, st->name);
fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", rrdset_id(st), rrdset_name(st));
fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME1\n");
rrddim_set_name(st, rd1, "DIM1NAME1");
fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd1->id, rd1->name);
fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd1), rrddim_name(rd1));
fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME2\n");
rrddim_set_name(st, rd1, "DIM1NAME2");
fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd1->id, rd1->name);
fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd1), rrddim_name(rd1));
fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME1\n");
rrddim_set_name(st, rd2, "DIM2NAME1");
fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd2->id, rd2->name);
fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2));
fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME2\n");
rrddim_set_name(st, rd2, "DIM2NAME2");
fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd2->id, rd2->name);
fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2));
BUFFER *buf = buffer_create(1);
health_api_v1_chart_variables2json(st, buf);
@ -1449,7 +1449,7 @@ int unit_test(long delay, long shift)
unsigned long c, dimensions = 0;
RRDDIM *rd;
for(rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
for(rd = st->dimensions; rd ; rd = rd->next) dimensions++;
for(c = 0; c < 20 ;c++) {
i += increment;
@ -1470,7 +1470,7 @@ int unit_test(long delay, long shift)
}
// prevent it from deleting the dimensions
for(rd = st->dimensions ; rd ; rd = rd->next)
for(rd = st->dimensions; rd ; rd = rd->next)
rd->last_collected_time.tv_sec = st->last_collected_time.tv_sec;
rrdset_done(st);
@ -1486,10 +1486,10 @@ int unit_test(long delay, long shift)
for(c = 0 ; c < st->counter ; c++) {
fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10);
for(rd = st->dimensions ; rd ; rd = rd->next) {
for(rd = st->dimensions; rd ; rd = rd->next) {
sn = rd->db[c];
cn = unpack_storage_number(sn);
fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rd->id, cn, sn);
fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rrddim_id(rd), cn, sn);
if(rd == rdabs) v =
( oincrement
@ -1909,14 +1909,14 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
if(!value_errors)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, expected, value);
rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now + k * update_every, expected, value);
value_errors++;
errors++;
}
if(end_time != time_now + k * update_every) {
if(!time_errors)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, (unsigned long)time_retrieved);
rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now + k * update_every, (unsigned long)time_retrieved);
time_errors++;
errors++;
}
@ -1957,7 +1957,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
NULL, NULL, NULL, 0, 0);
if (!r) {
fprintf(stderr, " DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", st[i]->name, current_region);
fprintf(stderr, " DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", rrdset_name(st[i]), current_region);
return ++errors;
} else {
assert(r->st == st[i]);
@ -1967,7 +1967,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
time_retrieved = r->t[c];
// for each dimension
for (j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) {
for (j = 0, d = r->st->dimensions; d && j < r->d ; ++j, d = d->next) {
NETDATA_DOUBLE *cn = &r->v[ c * r->d ];
value = cn[j];
assert(rd[i][j] == d);
@ -1980,13 +1980,13 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
if(value_errors < 20)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, expected, value);
value_errors++;
}
if(time_retrieved != time_now) {
if(time_errors < 20)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved);
rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, (unsigned long)time_retrieved);
time_errors++;
}
}
@ -2090,7 +2090,7 @@ int test_dbengine(void)
time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0,
RRDR_OPTION_NATURAL_POINTS, NULL, NULL, NULL, 0, 0);
if (!r) {
fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", rrdset_name(st[i]));
++errors;
} else {
long c;
@ -2103,7 +2103,7 @@ int test_dbengine(void)
time_t time_retrieved = r->t[c];
// for each dimension
for(j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) {
for(j = 0, d = r->st->dimensions; d && j < r->d ; ++j, d = d->next) {
NETDATA_DOUBLE *cn = &r->v[ c * r->d ];
NETDATA_DOUBLE value = cn[j];
assert(rd[i][j] == d);
@ -2116,13 +2116,13 @@ int test_dbengine(void)
if(!value_errors)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, expected, value);
value_errors++;
}
if(time_retrieved != time_now) {
if(!time_errors)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved);
rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, (unsigned long)time_retrieved);
time_errors++;
}
}
@ -2350,7 +2350,7 @@ static void query_dbengine_chart(void *arg)
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", found data gap, ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected);
rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected);
++thread_info->errors;
}
break;
@ -2365,7 +2365,7 @@ static void query_dbengine_chart(void *arg)
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", found data gap, ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected);
rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected);
++thread_info->errors;
}
break;
@ -2378,7 +2378,7 @@ static void query_dbengine_chart(void *arg)
if(!value_errors)
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected, value);
rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected, value);
value_errors++;
thread_info->errors++;
}
@ -2388,7 +2388,7 @@ static void query_dbengine_chart(void *arg)
if(!time_errors)
fprintf(stderr,
" DB-engine stresstest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, (unsigned long) time_retrieved);
rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, (unsigned long) time_retrieved);
time_errors++;
thread_info->errors++;
}

4
database/engine/metadata_log/metalogpluginsd.c
View File

@ -12,11 +12,11 @@ PARSER_RC metalog_pluginsd_host_action(
{
struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
RRDHOST *host = rrdhost_find_by_guid(machine_guid);
if (host) {
if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {
error("Archived host '%s' has memory mode '%s', but the archived one is '%s'. Ignoring archived state.",
host->hostname, rrd_memory_mode_name(host->rrd_memory_mode),
rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode),
rrd_memory_mode_name(RRD_MEMORY_MODE_DBENGINE));
((PARSER_USER_OBJECT *) user)->host = NULL; /* Ignore objects if memory mode is not dbengine */
}

5
database/engine/rrdengineapi.c
View File

@ -98,7 +98,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_insta
pg_cache = &ctx->pg_cache;
rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid);
rrdeng_generate_legacy_uuid(rrddim_id(rd), (char *)rrdset_id(rd->rrdset), &legacy_uuid);
if (host != localhost && is_storage_engine_shared((STORAGE_INSTANCE *)ctx))
is_multihost_child = 1;
@ -138,8 +138,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_insta
uuid_copy(rd->metric_uuid, multihost_legacy_uuid);
if (unlikely(need_to_store && !ctx->tier))
(void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
rd->algorithm);
(void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm);
}
struct rrdeng_metric_handle *mh = mallocz(sizeof(struct rrdeng_metric_handle));

8
database/ram/rrddim_mem.c
View File

@ -91,7 +91,7 @@ static inline size_t rrddim_time2slot(RRDDIM *rd, time_t t) {
}
if(unlikely(ret >= entries)) {
error("INTERNAL ERROR: rrddim_time2slot() on %s returns values outside entries", rd->name);
error("INTERNAL ERROR: rrddim_time2slot() on %s returns values outside entries", rrddim_name(rd));
ret = entries - 1;
}
@ -119,12 +119,12 @@ static inline time_t rrddim_slot2time(RRDDIM *rd, size_t slot) {
ret = last_entry_t - (time_t)(update_every * (last_slot - slot));
if(unlikely(ret < first_entry_t)) {
error("INTERNAL ERROR: rrddim_slot2time() on %s returns time too far in the past", rd->name);
error("INTERNAL ERROR: rrddim_slot2time() on %s returns time too far in the past", rrddim_name(rd));
ret = first_entry_t;
}
if(unlikely(ret > last_entry_t)) {
error("INTERNAL ERROR: rrddim_slot2time() on %s returns time into the future", rd->name);
error("INTERNAL ERROR: rrddim_slot2time() on %s returns time into the future", rrddim_name(rd));
ret = last_entry_t;
}
@ -206,7 +206,7 @@ int rrddim_query_is_finished(struct rrddim_query_handle *handle) {
void rrddim_query_finalize(struct rrddim_query_handle *handle) {
#ifdef NETDATA_INTERNAL_CHECKS
if(!rrddim_query_is_finished(handle))
error("QUERY: query for chart '%s' dimension '%s' has been stopped unfinished", handle->rd->rrdset->id, handle->rd->name);
error("QUERY: query for chart '%s' dimension '%s' has been stopped unfinished", rrdset_id(handle->rd->rrdset), rrddim_name(handle->rd));
#endif
freez(handle->handle);
}

12
database/rrd.c
View File

@ -154,3 +154,15 @@ char *rrdset_cache_dir(RRDHOST *host, const char *id) {
return ret;
}
// ----------------------------------------------------------------------------
// RRD - string management
STRING *rrd_string_strdupz(const char *s) {
if(unlikely(!s || !*s)) return string_strdupz(s);
char *tmp = strdupz(s);
json_fix_string(tmp);
STRING *ret = string_strdupz(tmp);
freez(tmp);
return ret;
}

349
database/rrd.h
View File

@ -160,14 +160,10 @@ extern const char *rrd_algorithm_name(RRD_ALGORITHM algorithm);
// RRD FAMILY
struct rrdfamily {
avl_t avl;
const char *family;
uint32_t hash_family;
STRING *family;
DICTIONARY *rrdvar_root_index;
size_t use_count;
avl_tree_lock rrdvar_root_index;
};
typedef struct rrdfamily RRDFAMILY;
@ -190,6 +186,7 @@ typedef enum rrddim_flags {
RRDDIM_FLAG_PENDING_FOREACH_ALARM = (1 << 5), // set when foreach alarm has not been initialized yet
RRDDIM_FLAG_META_HIDDEN = (1 << 6), // Status of hidden option in the metadata database
RRDDIM_FLAG_INDEXED_ID = (1 << 7),
} RRDDIM_FLAGS;
#define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_SEQ_CST) & (flag))
@ -244,27 +241,13 @@ extern bool exporting_labels_filter_callback(const char *name, const char *value
// RRD DIMENSION - this is a metric
struct rrddim {
// ------------------------------------------------------------------------
// binary indexing structures
avl_t avl; // the binary index - this has to be first member!
uuid_t metric_uuid; // global UUID for this metric (unique_across hosts)
// ------------------------------------------------------------------------
// the dimension definition
const char *id; // the id of this dimension (for internal identification)
const char *name; // the name of this dimension (as presented to user)
// this is a pointer to the config structure
// since the config always has a higher priority
// (the user overwrites the name of the charts)
uint32_t hash; // a simple hash of the id, to speed up searching / indexing
// instead of strcmp() every item in the binary index
// we first compare the hashes
uint32_t hash_name; // a simple hash of the name
STRING *id; // the id of this dimension (for internal identification)
STRING *name; // the name of this dimension (as presented to user)
RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
RRDDIM_FLAGS flags; // configuration flags for the dimension
@ -305,6 +288,8 @@ struct rrddim {
NETDATA_DOUBLE stored_volume; // the sum of all stored values so far
struct rrddim *next; // linking of dimensions within the same data set
struct rrddim *prev; // linking of dimensions within the same data set
struct rrdset *rrdset;
RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
@ -328,6 +313,9 @@ struct rrddim {
storage_number *db; // the array of values
};
#define rrddim_id(rd) string2str((rd)->id)
#define rrddim_name(rd) string2str((rd) ->name)
// returns the RRDDIM cache filename, or NULL if it does not exist
extern const char *rrddim_cache_filename(RRDDIM *rd);
@ -459,9 +447,6 @@ extern void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t n
// ----------------------------------------------------------------------------
// volatile state per chart
struct rrdset_volatile {
char *old_title;
char *old_units;
char *old_context;
uuid_t hash_id;
DICTIONARY *chart_labels;
bool is_ar_chart;
@ -485,88 +470,75 @@ struct rrdset_volatile {
// and may lead to missing information.
typedef enum rrdset_flags {
RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another
// (the master data set should be the one that has the same family and is not detail)
RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart
RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete
RRDSET_FLAG_EXPORTING_SEND = 1 << 4, // if set, this chart should be sent to Prometheus web API and external databases
RRDSET_FLAG_EXPORTING_IGNORE = 1 << 5, // if set, this chart should not be sent to Prometheus web API and external databases
RRDSET_FLAG_UPSTREAM_SEND = 1 << 6, // if set, this chart should be sent upstream (streaming)
RRDSET_FLAG_UPSTREAM_IGNORE = 1 << 7, // if set, this chart should not be sent upstream (streaming)
RRDSET_FLAG_UPSTREAM_EXPOSED = 1 << 8, // if set, we have sent this chart definition to netdata parent (streaming)
RRDSET_FLAG_STORE_FIRST = 1 << 9, // if set, do not eliminate the first collection during interpolation
RRDSET_FLAG_HETEROGENEOUS = 1 << 10, // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
RRDSET_FLAG_HOMOGENEOUS_CHECK = 1 << 11, // if set, the chart should be checked to determine if the dimensions are homogeneous
RRDSET_FLAG_HIDDEN = 1 << 12, // if set, do not show this chart on the dashboard, but use it for exporting
RRDSET_FLAG_SYNC_CLOCK = 1 << 13, // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14, // this is marked by the collector/module when a chart has obsolete dimensions
// No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at
// least rrdset_free_obsolete_time seconds ago.
RRDSET_FLAG_ARCHIVED = 1 << 15,
RRDSET_FLAG_ACLK = 1 << 16,
RRDSET_FLAG_PENDING_FOREACH_ALARMS = 1 << 17, // contains dims with uninitialized foreach alarms
RRDSET_FLAG_ANOMALY_DETECTION = 1 << 18 // flag to identify anomaly detection charts.
RRDSET_FLAG_DETAIL = (1 << 1), // if set, the data set should be considered as a detail of another
// (the master data set should be the one that has the same family and is not detail)
RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart
RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete
RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases
RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases
RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming)
RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming)
RRDSET_FLAG_UPSTREAM_EXPOSED = (1 << 8), // if set, we have sent this chart definition to netdata parent (streaming)
RRDSET_FLAG_STORE_FIRST = (1 << 9), // if set, do not eliminate the first collection during interpolation
RRDSET_FLAG_HETEROGENEOUS = (1 << 10), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 11), // if set, the chart should be checked to determine if the dimensions are homogeneous
RRDSET_FLAG_HIDDEN = (1 << 12), // if set, do not show this chart on the dashboard, but use it for exporting
RRDSET_FLAG_SYNC_CLOCK = (1 << 13), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 14), // this is marked by the collector/module when a chart has obsolete dimensions
// No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at
// least rrdset_free_obsolete_time seconds ago.
RRDSET_FLAG_ARCHIVED = (1 << 15),
RRDSET_FLAG_ACLK = (1 << 16),
RRDSET_FLAG_PENDING_FOREACH_ALARMS = (1 << 17), // contains dims with uninitialized foreach alarms
RRDSET_FLAG_ANOMALY_DETECTION = (1 << 18), // flag to identify anomaly detection charts.
RRDSET_FLAG_INDEXED_ID = (1 << 19),
RRDSET_FLAG_INDEXED_NAME = (1 << 20),
} RRDSET_FLAGS;
#define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_SEQ_CST) & (flag))
#define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_SEQ_CST)
#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~flag, __ATOMIC_SEQ_CST)
#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~(flag), __ATOMIC_SEQ_CST)
struct rrdset {
// ------------------------------------------------------------------------
// binary indexing structures
avl_t avl; // the index, with key the id - this has to be first!
avl_t avlname; // the index, with key the name
// ------------------------------------------------------------------------
// the set configuration
char id[RRD_ID_LENGTH_MAX + 1]; // id of the data set
const char *name; // the name of this dimension (as presented to user)
// this is a pointer to the config structure
// since the config always has a higher priority
// (the user overwrites the name of the charts)
char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options)
char *family; // grouping sets under the same family
char *title; // title shown to user
char *units; // units of measurement
char *context; // the template of this data set
uint32_t hash_context; // the hash of the chart's context
STRING *id; // the ID of the data set
STRING *name; // the name of this dimension (as presented to user)
STRING *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options)
STRING *family; // grouping sets under the same family
STRING *title; // title shown to user
STRING *units; // units of measurement
STRING *context; // the template of this data set
STRING *plugin_name; // the name of the plugin that generated this
STRING *module_name; // the name of the plugin module that generated this
RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart
RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to
RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset
RRDSET_TYPE chart_type; // line, area, stacked
RRDSET_FLAGS flags; // configuration flags
RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances
int update_every; // every how many seconds is this updated?
int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored
// netdata will interpolate values for gaps lower than this
long entries; // total number of entries in the data set
long current_entry; // the entry that is currently being updated
// it goes around in a round-robin fashion
RRDSET_FLAGS flags; // configuration flags
RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances
int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored
// netdata will interpolate values for gaps lower than this
long priority; // the sorting priority of this chart
// ------------------------------------------------------------------------
// members for temporary data we need for calculations
RRD_MEMORY_MODE rrd_memory_mode; // if set to 1, this is memory mapped
char *cache_dir; // the directory to store dimensions
netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list
size_t counter; // the number of times we added values to this database
size_t counter_done; // the number of times rrdset_done() has been called
@ -576,19 +548,12 @@ struct rrdset {
};
time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream
char *plugin_name; // the name of the plugin that generated this
char *module_name; // the name of the plugin module that generated this
uuid_t *chart_uuid; // Store the global GUID for this chart
// this object.
struct rrdset_volatile *state; // volatile state that is not persistently stored
size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine
uint32_t hash; // a simple hash on the id, to speed up searching
// we first compare hashes, and only if the hashes are equal we do string comparisons
uint32_t hash_name; // a simple hash on the name
usec_t usec_since_last_update; // the time in microseconds since the last collection of data
struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function)
@ -601,14 +566,15 @@ struct rrdset {
RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to
struct rrdset *next; // linking of rrdsets
struct rrdset *prev; // linking of rrdsets
// ------------------------------------------------------------------------
// local variables
NETDATA_DOUBLE green; // green threshold for this chart
NETDATA_DOUBLE red; // red threshold for this chart
NETDATA_DOUBLE green; // green threshold for this chart
NETDATA_DOUBLE red; // red threshold for this chart
avl_tree_lock rrdvar_root_index; // RRDVAR index for this chart
DICTIONARY *rrdvar_root_index; // RRDVAR index for this chart
RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs)
RRDCALC *alarms; // RRDCALC linked list for this chart
@ -621,14 +587,27 @@ struct rrdset {
// ------------------------------------------------------------------------
// the dimensions
avl_tree_lock dimensions_index; // the root of the dimensions index
DICTIONARY *rrddim_root_index; // the root of the dimensions index
netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list
RRDDIM *dimensions; // the actual data for every dimension
};
#define rrdset_plugin_name(st) string2str((st)->plugin_name)
#define rrdset_module_name(st) string2str((st)->module_name)
#define rrdset_units(st) string2str((st)->units)
#define rrdset_type(st) string2str((st)->type)
#define rrdset_family(st) string2str((st)->family)
#define rrdset_title(st) string2str((st)->title)
#define rrdset_context(st) string2str((st)->context)
#define rrdset_name(st) string2str((st)->name)
#define rrdset_id(st) string2str((st)->id)
#define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock))
#define rrdset_wrlock(st) netdata_rwlock_wrlock(&((st)->rrdset_rwlock))
#define rrdset_unlock(st) netdata_rwlock_unlock(&((st)->rrdset_rwlock))
extern STRING *rrd_string_strdupz(const char *s);
// ----------------------------------------------------------------------------
// these loop macros make sure the linked list is accessed with the right lock
@ -653,26 +632,29 @@ extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY
// and may lead to missing information.
typedef enum rrdhost_flags {
RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data)
RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts
RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan
RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases
RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases
RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet
RRDHOST_FLAG_MULTIHOST = (1 << 6), // Host belongs to localhost/megadb
RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms
RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8),
RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9),
RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates
RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data)
RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts
RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan
RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases
RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases
RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet
RRDHOST_FLAG_MULTIHOST = (1 << 6), // Host belongs to localhost/megadb
RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms
RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8),
RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9),
RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates
RRDHOST_FLAG_INDEXED_MACHINE_GUID = (1 << 11), // when set, we have indexed its machine guid
RRDHOST_FLAG_INDEXED_HOSTNAME = (1 << 12), // when set, we have indexed its hostname
RRDHOST_FLAG_STREAM_COLLECTED_METRICS = (1 << 13), // when set, rrdset_done() should push metrics to parent
} RRDHOST_FLAGS;
#define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag))
#define rrdhost_flag_set(host, flag) __atomic_or_fetch(&((host)->flags), flag, __ATOMIC_SEQ_CST)
#define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~flag, __ATOMIC_SEQ_CST)
#define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~(flag), __ATOMIC_SEQ_CST)
#ifdef NETDATA_INTERNAL_CHECKS
#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, st->name, ##args); } while(0)
debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0)
#else
#define rrdset_debug(st, fmt, args...) debug_dummy()
#endif
@ -690,34 +672,30 @@ struct alarm_entry {
time_t duration;
time_t non_clear_duration;
char *name;
uint32_t hash_name;
STRING *name;
STRING *chart;
STRING *chart_context;
STRING *family;
char *chart;
uint32_t hash_chart;
char *chart_context;
STRING *classification;
STRING *component;
STRING *type;
char *family;
char *classification;
char *component;
char *type;
char *exec;
char *recipient;
STRING *exec;
STRING *recipient;
time_t exec_run_timestamp;
int exec_code;
uint64_t exec_spawn_serial;
char *source;
char *units;
char *info;
STRING *source;
STRING *units;
STRING *info;
NETDATA_DOUBLE old_value;
NETDATA_DOUBLE new_value;
char *old_value_string;
char *new_value_string;
STRING *old_value_string;
STRING *new_value_string;
RRDCALC_STATUS old_status;
RRDCALC_STATUS new_status;
@ -737,6 +715,20 @@ struct alarm_entry {
struct alarm_entry *prev_in_progress;
};
#define ae_name(ae) string2str((ae)->name)
#define ae_chart_name(ae) string2str((ae)->chart)
#define ae_chart_context(ae) string2str((ae)->chart_context)
#define ae_family(ae) string2str((ae)->family)
#define ae_classification(ae) string2str((ae)->classification)
#define ae_component(ae) string2str((ae)->component)
#define ae_type(ae) string2str((ae)->type)
#define ae_exec(ae) string2str((ae)->exec)
#define ae_recipient(ae) string2str((ae)->recipient)
#define ae_source(ae) string2str((ae)->source)
#define ae_units(ae) string2str((ae)->units)
#define ae_info(ae) string2str((ae)->info)
#define ae_old_value_string(ae) string2str((ae)->old_value_string)
#define ae_new_value_string(ae) string2str((ae)->new_value_string)
typedef struct alarm_log {
uint32_t next_log_id;
@ -790,24 +782,20 @@ struct rrdhost_system_info {
};
struct rrdhost {
avl_t avl; // the index of hosts
char machine_guid[GUID_LEN + 1]; // the unique ID of this host
// ------------------------------------------------------------------------
// host information
char *hostname; // the hostname of this host
uint32_t hash_hostname; // the hostname hash
STRING *hostname; // the hostname of this host
STRING *registry_hostname; // the registry hostname for this host
STRING *os; // the O/S type of the host
STRING *tags; // tags for this host
STRING *timezone; // the timezone of the host
STRING *abbrev_timezone; // the abbriviated timezone of the host
STRING *program_name; // the program name that collects metrics for this host
STRING *program_version; // the program version that collects metrics for this host
char *registry_hostname; // the registry hostname for this host
char machine_guid[GUID_LEN + 1]; // the unique ID of this host
uint32_t hash_machine_guid; // the hash of the unique ID
const char *os; // the O/S type of the host
const char *tags; // tags for this host
const char *timezone; // the timezone of the host
const char *abbrev_timezone; // the abbriviated timezone of the host
int32_t utc_offset; // the offset in seconds from utc
RRDHOST_FLAGS flags; // flags about this RRDHOST
@ -820,9 +808,6 @@ struct rrdhost {
char *cache_dir; // the directory to save RRD cache files
char *varlib_dir; // the directory to save health log
char *program_name; // the program name that collects metrics for this host
char *program_version; // the program version that collects metrics for this host
struct rrdhost_system_info *system_info; // information collected from the host environment
// ------------------------------------------------------------------------
@ -873,23 +858,21 @@ struct rrdhost {
// health monitoring options
unsigned int health_enabled; // 1 when this host has health enabled
time_t health_delay_up_to; // a timestamp to delay alarms processing up to
char *health_default_exec; // the full path of the alarms notifications program
char *health_default_recipient; // the default recipient for all alarms
char *health_log_filename; // the alarms event log filename
size_t health_log_entries_written; // the number of alarm events written to the alarms event log
FILE *health_log_fp; // the FILE pointer to the open alarms event log file
uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications
uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications
time_t health_delay_up_to; // a timestamp to delay alarms processing up to
STRING *health_default_exec; // the full path of the alarms notifications program
STRING *health_default_recipient; // the default recipient for all alarms
char *health_log_filename; // the alarms event log filename
size_t health_log_entries_written; // the number of alarm events written to the alarms event log
FILE *health_log_fp; // the FILE pointer to the open alarms event log file
uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications
uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications
// all RRDCALCs are primarily allocated and linked here
// RRDCALCs may be linked to charts at any point
// (charts may or may not exist when these are loaded)
RRDCALC *alarms;
RRDCALC *host_alarms;
RRDCALC *alarms_with_foreach;
avl_tree_lock alarms_idx_health_log;
avl_tree_lock alarms_idx_name;
ALARM_LOG health_log; // alarms historical events (event log)
uint32_t health_last_processed_id; // the last processed health id from the log
@ -899,9 +882,7 @@ struct rrdhost {
// templates of alarms
// these are used to create alarms when charts
// are created or renamed, that match them
RRDCALCTEMPLATE *templates;
RRDCALCTEMPLATE *alarms_template_with_foreach;
RRDCALCTEMPLATE *alarms_templates;
// ------------------------------------------------------------------------
// the charts of the host
@ -927,11 +908,11 @@ struct rrdhost {
// ------------------------------------------------------------------------
// indexes
avl_tree_lock rrdset_root_index; // the host's charts index (by id)
avl_tree_lock rrdset_root_index_name; // the host's charts index (by name)
DICTIONARY *rrdset_root_index; // the host's charts index (by id)
DICTIONARY *rrdset_root_index_name; // the host's charts index (by name)
avl_tree_lock rrdfamily_root_index; // the host's chart families index
avl_tree_lock rrdvar_root_index; // the host's chart variables index
DICTIONARY *rrdfamily_root_index; // the host's chart families index
DICTIONARY *rrdvar_root_index; // the host's chart variables index
STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers
@ -950,9 +931,19 @@ struct rrdhost {
aclk_rrdhost_state aclk_state;
struct rrdhost *next;
struct rrdhost *prev;
};
extern RRDHOST *localhost;
#define rrdhost_hostname(host) string2str((host)->hostname)
#define rrdhost_registry_hostname(host) string2str((host)->registry_hostname)
#define rrdhost_os(host) string2str((host)->os)
#define rrdhost_tags(host) string2str((host)->tags)
#define rrdhost_timezone(host) string2str((host)->timezone)
#define rrdhost_abbrev_timezone(host) string2str((host)->abbrev_timezone)
#define rrdhost_program_name(host) string2str((host)->program_name)
#define rrdhost_program_version(host) string2str((host)->program_version)
#define rrdhost_rdlock(host) netdata_rwlock_rdlock(&((host)->rrdhost_rwlock))
#define rrdhost_wrlock(host) netdata_rwlock_wrlock(&((host)->rrdhost_rwlock))
#define rrdhost_unlock(host) netdata_rwlock_unlock(&((host)->rrdhost_rwlock))
@ -960,6 +951,8 @@ extern RRDHOST *localhost;
#define rrdhost_aclk_state_lock(host) netdata_mutex_lock(&((host)->aclk_state_lock))
#define rrdhost_aclk_state_unlock(host) netdata_mutex_unlock(&((host)->aclk_state_lock))
extern long rrdhost_hosts_available(void);
// ----------------------------------------------------------------------------
// these loop macros make sure the linked list is accessed with the right lock
@ -990,8 +983,8 @@ extern time_t rrdhost_free_orphan_time;
extern int rrd_init(char *hostname, struct rrdhost_system_info *system_info);
extern RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash);
extern RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash);
extern RRDHOST *rrdhost_find_by_hostname(const char *hostname);
extern RRDHOST *rrdhost_find_by_guid(const char *guid);
extern RRDHOST *rrdhost_find_or_create(
const char *hostname
@ -1237,11 +1230,18 @@ time_t rrdhost_last_entry_t(RRDHOST *h);
// RRD DIMENSION functions
extern void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host);
extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier,
collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode);//,
//int is_archived, uuid_t *dim_uuid);
#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, \
algorithm, (st)->rrd_memory_mode)//, 0, NULL)
extern RRDDIM *rrddim_add_custom(RRDSET *st
, const char *id
, const char *name
, collected_number multiplier
, collected_number divisor
, RRD_ALGORITHM algorithm
, RRD_MEMORY_MODE memory_mode
);
#define rrddim_add(st, id, name, multiplier, divisor, algorithm) \
rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name);
extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
@ -1249,15 +1249,7 @@ extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip
extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor);
extern RRDDIM *rrddim_find(RRDSET *st, const char *id);
/* This will not return dimensions that are archived */
static inline RRDDIM *rrddim_find_active(RRDSET *st, const char *id)
{
RRDDIM *rd = rrddim_find(st, id);
if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
return NULL;
return rd;
}
extern RRDDIM *rrddim_find_active(RRDSET *st, const char *id);
extern int rrddim_hide(RRDSET *st, const char *id);
extern int rrddim_unhide(RRDSET *st, const char *id);
@ -1275,33 +1267,20 @@ extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);
// ----------------------------------------------------------------------------
// Miscellaneous functions
extern int alarm_compare_id(void *a, void *b);
extern int alarm_compare_name(void *a, void *b);
extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length);
// ----------------------------------------------------------------------------
// RRD internal functions
#ifdef NETDATA_RRD_INTERNALS
extern avl_tree_lock rrdhost_root_index;
extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length);
extern char *rrdset_cache_dir(RRDHOST *host, const char *id);
extern void rrddim_free(RRDSET *st, RRDDIM *rd);
extern int rrddim_compare(void* a, void* b);
extern int rrdset_compare(void* a, void* b);
extern int rrdset_compare_name(void* a, void* b);
extern int rrdfamily_compare(void *a, void *b);
extern RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id);
extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc);
#define rrdset_index_add(host, st) (RRDSET *)avl_insert_lock(&((host)->rrdset_root_index), (avl_t *)(st))
#define rrdset_index_del(host, st) (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index), (avl_t *)(st))
extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st);
extern void rrdset_free(RRDSET *st);
extern void rrdset_reset(RRDSET *st);
extern void rrdset_save(RRDSET *st);
@ -1318,8 +1297,8 @@ extern RRDHOST *rrdhost_create(
#endif /* NETDATA_RRD_INTERNALS */
extern void set_host_properties(
RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *hostname, const char *registry_hostname,
const char *guid, const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname,
const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
const char *program_name, const char *program_version);
extern int get_tier_grouping(int tier);

454
database/rrdcalc.c
View File

@ -35,9 +35,8 @@ inline const char *rrdcalc_status2string(RRDCALC_STATUS status) {
}
}
char *rrdcalc_replace_variables(const char *line, RRDCALC *rc)
{
if (!line)
static STRING *rrdcalc_replace_variables(const char *line, RRDCALC *rc) {
if (!line || !*line)
return NULL;
size_t pos = 0;
@ -46,24 +45,28 @@ char *rrdcalc_replace_variables(const char *line, RRDCALC *rc)
char *m, *lbl_value = NULL;
while ((m = strchr(temp + pos, '$'))) {
int i=0;
int i = 0;
char *e = m;
while (*e) {
if (*e == ' ' || i == RRDCALC_VAR_MAX - 1) {
if (*e == ' ' || i == RRDCALC_VAR_MAX - 1)
break;
}
else
var[i]=*e;
var[i] = *e;
e++;
i++;
}
var[i]='\0';
var[i] = '\0';
pos = m - temp + 1;
if (!strcmp(var, RRDCALC_VAR_FAMILY)) {
char *buf = find_and_replace(temp, var, (rc->rrdset && rc->rrdset->family) ? rc->rrdset->family : "", m);
char *buf = find_and_replace(temp, var, (rc->rrdset && rc->rrdset->family) ? rrdset_family(rc->rrdset) : "", m);
freez(temp);
temp = buf;
} else if (!strncmp(var, RRDCALC_VAR_LABEL, RRDCALC_VAR_LABEL_LEN)) {
}
else if (!strncmp(var, RRDCALC_VAR_LABEL, RRDCALC_VAR_LABEL_LEN)) {
if(likely(rc->rrdset && rc->rrdset->state && rc->rrdset->state->chart_labels)) {
rrdlabels_get_value_to_char_or_null(rc->rrdset->state->chart_labels, &lbl_value, var+RRDCALC_VAR_LABEL_LEN);
if (lbl_value) {
@ -76,17 +79,20 @@ char *rrdcalc_replace_variables(const char *line, RRDCALC *rc)
}
}
return temp;
STRING *ret = string_strdupz(temp);
freez(temp);
return ret;
}
void rrdcalc_update_rrdlabels(RRDSET *st) {
RRDCALC *rc;
for( rc = st->alarms; rc ; rc = rc->rrdset_next ) {
foreach_rrdcalc_in_rrdset(st, rc) {
if (rc->original_info) {
if (rc->info)
freez(rc->info);
string_freez(rc->info);
rc->info = rrdcalc_replace_variables(rc->original_info, rc);
rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc);
}
}
}
@ -94,58 +100,57 @@ void rrdcalc_update_rrdlabels(RRDSET *st) {
static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
RRDHOST *host = st->rrdhost;
debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);
debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host));
rc->last_status_change = now_realtime_sec();
rc->rrdset = st;
rc->rrdset_next = st->alarms;
rc->rrdset_prev = NULL;
if(rc->rrdset_next)
rc->rrdset_next->rrdset_prev = rc;
st->alarms = rc;
DOUBLE_LINKED_LIST_PREPEND_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next);
if(rc->update_every < rc->rrdset->update_every) {
error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rc->rrdset->id, rc->name, rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every);
error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every);
rc->update_every = rc->rrdset->update_every;
}
if(!isnan(rc->green) && isnan(st->green)) {
debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " NETDATA_DOUBLE_FORMAT_AUTO
" to " NETDATA_DOUBLE_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green);
" to " NETDATA_DOUBLE_FORMAT_AUTO ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->green, rc->green);
st->green = rc->green;
}
if(!isnan(rc->red) && isnan(st->red)) {
debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " NETDATA_DOUBLE_FORMAT_AUTO " to " NETDATA_DOUBLE_FORMAT_AUTO
".", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red);
".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->red, rc->red);
st->red = rc->red;
}
rc->local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value);
rc->family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value);
rc->local = rrdvar_create_and_index("local", st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value);
rc->family = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value);
char fullname[RRDVAR_MAX_LENGTH + 1];
snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name);
rc->hostid = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value);
snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), rrdcalc_name(rc));
STRING *fullname_string = string_strdupz(fullname);
rc->hostid = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value);
snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name);
rc->hostname = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value);
snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), rrdcalc_name(rc));
rc->hostname = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value);
string_freez(fullname_string);
if(rc->hostid && !rc->hostname)
rc->hostid->options |= RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR;
if(!rc->units) rc->units = strdupz(st->units);
if(!rc->units) rc->units = string_dup(st->units);
if (rc->original_info) {
if (rc->info)
freez(rc->info);
rc->info = rrdcalc_replace_variables(rc->original_info, rc);
string_freez(rc->info);
rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc);
}
time_t now = now_realtime_sec();
ALARM_ENTRY *ae = health_create_alarm_entry(
host,
rc->id,
@ -170,19 +175,20 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
rc->units,
rc->info,
0,
0);
rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
health_alarm_log(host, ae);
}
static int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) {
if((rc->hash_chart != st->hash || strcmp(rc->chart, st->id) != 0) &&
(rc->hash_chart != st->hash_name || strcmp(rc->chart, st->name) != 0))
static inline int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) {
if ( (rc->chart != st->id)
&& (rc->chart != st->name))
return 0;
if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, st->module_name))
if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, rrdset_module_name(st)))
return 0;
if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, st->plugin_name))
if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, rrdset_plugin_name(st)))
return 0;
if (st->rrdhost->host_labels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->host_labels, rc->host_labels_pattern, '='))
@ -197,7 +203,7 @@ inline void rrdsetcalc_link_matching(RRDSET *st) {
// debug(D_HEALTH, "find matching alarms for chart '%s'", st->id);
RRDCALC *rc;
for(rc = host->alarms; rc ; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
if(unlikely(rc->rrdset))
continue;
@ -211,14 +217,15 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
RRDSET *st = rc->rrdset;
if(!st) {
debug(D_HEALTH, "Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name);
error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name);
debug(D_HEALTH, "Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rrdcalc_chart_name(rc), rrdcalc_name(rc));
error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rrdcalc_chart_name(rc), rrdcalc_name(rc));
return;
}
RRDHOST *host = st->rrdhost;
time_t now = now_realtime_sec();
ALARM_ENTRY *ae = health_create_alarm_entry(
host,
rc->id,
@ -244,32 +251,24 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
rc->info,
0,
0);
health_alarm_log(host, ae);
debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);
debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host));
// unlink it
if(rc->rrdset_prev)
rc->rrdset_prev->rrdset_next = rc->rrdset_next;
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next);
if(rc->rrdset_next)
rc->rrdset_next->rrdset_prev = rc->rrdset_prev;
if(st->alarms == rc)
st->alarms = rc->rrdset_next;
rc->rrdset_prev = rc->rrdset_next = NULL;
rrdvar_free(host, &st->rrdvar_root_index, rc->local);
rrdvar_free(host, st->rrdvar_root_index, rc->local);
rc->local = NULL;
rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rc->family);
rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rc->family);
rc->family = NULL;
rrdvar_free(host, &host->rrdvar_root_index, rc->hostid);
rrdvar_free(host, host->rrdvar_root_index, rc->hostid);
rc->hostid = NULL;
rrdvar_free(host, &host->rrdvar_root_index, rc->hostname);
rrdvar_free(host, host->rrdvar_root_index, rc->hostname);
rc->hostname = NULL;
rc->rrdset = NULL;
@ -280,18 +279,21 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
}
RRDCALC *rrdcalc_find(RRDSET *st, const char *name) {
RRDCALC *rc;
uint32_t hash = simple_hash(name);
RRDCALC *rc = NULL;
for( rc = st->alarms; rc ; rc = rc->rrdset_next ) {
if(unlikely(rc->hash == hash && !strcmp(rc->name, name)))
return rc;
STRING *name_string = string_strdupz(name);
foreach_rrdcalc_in_rrdset(st, rc) {
if(unlikely(rc->name == name_string))
break;
}
return NULL;
string_freez(name_string);
return rc;
}
inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name) {
inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name) {
RRDCALC *rc;
if(unlikely(!chart)) {
@ -299,36 +301,39 @@ inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, ui
return 1;
}
if(unlikely(!hash_chart)) hash_chart = simple_hash(chart);
if(unlikely(!hash_name)) hash_name = simple_hash(name);
STRING *name_string = string_strdupz(name);
STRING *chart_string = string_strdupz(chart);
int ret = 0;
// make sure it does not already exist
for(rc = host->alarms; rc ; rc = rc->next) {
if (unlikely(rc->chart && rc->hash == hash_name && rc->hash_chart == hash_chart && !strcmp(name, rc->name) && !strcmp(chart, rc->chart))) {
debug(D_HEALTH, "Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
info("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
return 1;
foreach_rrdcalc_in_rrdhost(host, rc) {
if (unlikely(rc->chart == chart_string && rc->name == name_string)) {
debug(D_HEALTH, "Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host));
info("Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host));
ret = 1;
break;
}
}
return 0;
string_freez(name_string);
string_freez(chart_string);
return ret;
}
inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id) {
if(chart && name) {
uint32_t hash_chart = simple_hash(chart);
uint32_t hash_name = simple_hash(name);
// re-use old IDs, by looking them up in the alarm log
ALARM_ENTRY *ae;
for(ae = host->health_log.alarms; ae ;ae = ae->next) {
if(unlikely(ae->hash_name == hash_name && ae->hash_chart == hash_chart && !strcmp(name, ae->name) && !strcmp(chart, ae->chart))) {
if(next_event_id) *next_event_id = ae->alarm_event_id + 1;
return ae->alarm_id;
}
inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id) {
// re-use old IDs, by looking them up in the alarm log
ALARM_ENTRY *ae = NULL;
for(ae = host->health_log.alarms; ae ;ae = ae->next) {
if(unlikely(name == ae->name && chart == ae->chart)) {
if(next_event_id) *next_event_id = ae->alarm_event_id + 1;
break;
}
}
if(ae)
return ae->alarm_id;
if (unlikely(!host->health_log.next_alarm_id))
host->health_log.next_alarm_id = (uint32_t)now_realtime_sec();
@ -347,7 +352,7 @@ inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const ch
*
* @return It returns the new name on success and the old otherwise
*/
char *alarm_name_with_dim(char *name, size_t namelen, const char *dim, size_t dimlen) {
char *alarm_name_with_dim(const char *name, size_t namelen, const char *dim, size_t dimlen) {
char *newname,*move;
newname = mallocz(namelen + dimlen + 2);
@ -407,15 +412,7 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) {
if(!rc->foreachdim) {
// link it to the host alarms list
if(likely(host->alarms)) {
// append it
RRDCALC *t;
for(t = host->alarms; t && t->next ; t = t->next) ;
t->next = rc;
}
else {
host->alarms = rc;
}
DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->host_alarms, rc, prev, next);
// link it to its chart
RRDSET *st;
@ -425,43 +422,33 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) {
break;
}
}
} else {
//link it case there is a foreach
if(likely(host->alarms_with_foreach)) {
// append it
RRDCALC *t;
for(t = host->alarms_with_foreach; t && t->next ; t = t->next) ;
t->next = rc;
}
else {
host->alarms_with_foreach = rc;
}
//I am not linking this alarm direct to the host here, this will be done when the children is created
}
else {
DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->alarms_with_foreach, rc, prev, next);
// We are not linking this alarm directly to the host here
// It will eventually be done if it matches the dimensions
}
}
inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) {
debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rt->name);
debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rrdcalctemplate_name(rt));
if(rrdcalc_exists(host, chart, rt->name, 0, 0))
if(rrdcalc_exists(host, chart, rrdcalctemplate_name(rt)))
return NULL;
RRDCALC *rc = callocz(1, sizeof(RRDCALC));
rc->next_event_id = 1;
rc->name = strdupz(rt->name);
rc->hash = simple_hash(rc->name);
rc->chart = strdupz(chart);
rc->hash_chart = simple_hash(rc->chart);
rc->name = string_dup(rt->name);
rc->chart = string_strdupz(chart);
uuid_copy(rc->config_hash_id, rt->config_hash_id);
rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id);
if(rt->dimensions) rc->dimensions = strdupz(rt->dimensions);
if(rt->foreachdim) {
rc->foreachdim = strdupz(rt->foreachdim);
rc->spdim = health_pattern_from_foreach(rc->foreachdim);
}
rc->dimensions = string_dup(rt->dimensions);
rc->foreachdim = string_dup(rt->foreachdim);
if(rt->foreachdim)
rc->spdim = health_pattern_from_foreach(rrdcalc_foreachdim(rc));
rc->foreachcounter = rt->foreachcounter;
rc->green = rt->green;
@ -485,55 +472,53 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt,
rc->update_every = rt->update_every;
rc->options = rt->options;
if(rt->exec) rc->exec = strdupz(rt->exec);
if(rt->recipient) rc->recipient = strdupz(rt->recipient);
if(rt->source) rc->source = strdupz(rt->source);
if(rt->units) rc->units = strdupz(rt->units);
if(rt->info) {
rc->info = strdupz(rt->info);
rc->original_info = strdupz(rt->info);
}
rc->exec = string_dup(rt->exec);
rc->recipient = string_dup(rt->recipient);
rc->source = string_dup(rt->source);
rc->units = string_dup(rt->units);
rc->info = string_dup(rt->info);
rc->original_info = string_dup(rt->info);
if (rt->classification) rc->classification = strdupz(rt->classification);
if (rt->component) rc->component = strdupz(rt->component);
if (rt->type) rc->type = strdupz(rt->type);
rc->classification = string_dup(rt->classification);
rc->component = string_dup(rt->component);
rc->type = string_dup(rt->type);
if(rt->calculation) {
rc->calculation = expression_parse(rt->calculation->source, NULL, NULL);
if(!rc->calculation)
error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, rt->name, rt->calculation->source);
error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, rrdcalctemplate_name(rt), rt->calculation->source);
}
if(rt->warning) {
rc->warning = expression_parse(rt->warning->source, NULL, NULL);
if(!rc->warning)
error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, rt->name, rt->warning->source);
error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, rrdcalctemplate_name(rt), rt->warning->source);
}
if(rt->critical) {
rc->critical = expression_parse(rt->critical->source, NULL, NULL);
if(!rc->critical)
error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source);
error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rrdcalctemplate_name(rt), rt->critical->source);
}
debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO
", red " NETDATA_DOUBLE_FORMAT_AUTO
", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
(rc->chart)?rc->chart:"NOCHART",
rc->name,
(rc->exec)?rc->exec:"DEFAULT",
(rc->recipient)?rc->recipient:"DEFAULT",
rrdcalc_chart_name(rc),
rrdcalc_name(rc),
(rc->exec)?rrdcalc_exec(rc):"DEFAULT",
(rc->recipient)?rrdcalc_recipient(rc):"DEFAULT",
rc->green,
rc->red,
(int)rc->group,
rc->after,
rc->before,
rc->options,
(rc->dimensions)?rc->dimensions:"NONE",
(rc->foreachdim)?rc->foreachdim:"NONE",
(rc->dimensions)?rrdcalc_dimensions(rc):"NONE",
(rc->foreachdim)?rrdcalc_foreachdim(rc):"NONE",
rc->update_every,
(rc->calculation)?rc->calculation->parsed_as:"NONE",
(rc->warning)?rc->warning->parsed_as:"NONE",
(rc->critical)?rc->critical->parsed_as:"NONE",
rc->source,
rrdcalc_source(rc),
rc->delay_up_duration,
rc->delay_down_duration,
rc->delay_max_duration,
@ -543,13 +528,6 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt,
);
rrdcalc_add_to_host(host, rc);
if(!rt->foreachdim) {
RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl_t *)rc);
if (rdcmp != rc) {
error("Cannot insert the alarm index ID %s",rc->name);
}
}
return rc;
}
@ -570,14 +548,12 @@ inline RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const ch
RRDCALC *newrc = callocz(1, sizeof(RRDCALC));
newrc->next_event_id = 1;
newrc->id = rrdcalc_get_unique_id(host, rc->chart, name, &rc->next_event_id);
newrc->name = (char *)name;
newrc->hash = simple_hash(newrc->name);
newrc->chart = strdupz(rc->chart);
newrc->hash_chart = simple_hash(rc->chart);
newrc->name = string_strdupz(name);
newrc->chart = string_dup(rc->chart);
newrc->id = rrdcalc_get_unique_id(host, newrc->chart, newrc->name, &rc->next_event_id);
uuid_copy(newrc->config_hash_id, *((uuid_t *) &rc->config_hash_id));
newrc->dimensions = strdupz(dimension);
newrc->dimensions = string_strdupz(dimension);
newrc->foreachdim = NULL;
rc->foreachcounter++;
newrc->foreachcounter = rc->foreachcounter;
@ -603,33 +579,33 @@ inline RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const ch
newrc->update_every = rc->update_every;
newrc->options = rc->options;
if(rc->exec) newrc->exec = strdupz(rc->exec);
if(rc->recipient) newrc->recipient = strdupz(rc->recipient);
if(rc->source) newrc->source = strdupz(rc->source);
if(rc->units) newrc->units = strdupz(rc->units);
if(rc->info) newrc->info = strdupz(rc->info);
if(rc->original_info) newrc->original_info = strdupz(rc->original_info);
newrc->exec = string_dup(rc->exec);
newrc->recipient = string_dup(rc->recipient);
newrc->source = string_dup(rc->source);
newrc->units = string_dup(rc->units);
newrc->info = string_dup(rc->info);
newrc->original_info = string_dup(rc->original_info);
if (rc->classification) newrc->classification = strdupz(rc->classification);
if (rc->component) newrc->component = strdupz(rc->component);
if (rc->type) newrc->type = strdupz(rc->type);
newrc->classification = string_dup(rc->classification);
newrc->component = string_dup(rc->component);
newrc->type = string_dup(rc->type);
if(rc->calculation) {
newrc->calculation = expression_parse(rc->calculation->source, NULL, NULL);
if(!newrc->calculation)
error("Health alarm '%s.%s': failed to parse calculation expression '%s'", rc->chart, rc->name, rc->calculation->source);
error("Health alarm '%s.%s': failed to parse calculation expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->calculation->source);
}
if(rc->warning) {
newrc->warning = expression_parse(rc->warning->source, NULL, NULL);
if(!newrc->warning)
error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", rc->chart, rc->name, rc->warning->source);
error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->warning->source);
}
if(rc->critical) {
newrc->critical = expression_parse(rc->critical->source, NULL, NULL);
if(!newrc->critical)
error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", rc->chart, rc->name, rc->critical->source);
error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->critical->source);
}
return newrc;
@ -642,86 +618,48 @@ void rrdcalc_free(RRDCALC *rc) {
expression_free(rc->warning);
expression_free(rc->critical);
freez(rc->name);
freez(rc->chart);
freez(rc->family);
freez(rc->dimensions);
freez(rc->foreachdim);
freez(rc->exec);
freez(rc->recipient);
freez(rc->source);
freez(rc->units);
freez(rc->info);
freez(rc->original_info);
freez(rc->classification);
freez(rc->component);
freez(rc->type);
string_freez(rc->name);
string_freez(rc->chart);
string_freez(rc->dimensions);
string_freez(rc->foreachdim);
string_freez(rc->exec);
string_freez(rc->recipient);
string_freez(rc->source);
string_freez(rc->units);
string_freez(rc->info);
string_freez(rc->original_info);
string_freez(rc->classification);
string_freez(rc->component);
string_freez(rc->type);
string_freez(rc->host_labels);
string_freez(rc->module_match);
string_freez(rc->plugin_match);
simple_pattern_free(rc->spdim);
freez(rc->host_labels);
simple_pattern_free(rc->host_labels_pattern);
freez(rc->module_match);
simple_pattern_free(rc->module_pattern);
freez(rc->plugin_match);
simple_pattern_free(rc->plugin_pattern);
freez(rc->family);
freez(rc);
}
void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) {
if(unlikely(!rc)) return;
debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdhost_hostname(host));
// unlink it from RRDSET
if(rc->rrdset) rrdsetcalc_unlink(rc);
// unlink it from RRDHOST
if(unlikely(rc == host->alarms))
host->alarms = rc->next;
else {
RRDCALC *t;
for(t = host->alarms; t && t->next != rc; t = t->next) ;
if(t) {
t->next = rc->next;
rc->next = NULL;
}
else
error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
}
RRDCALC *rdcmp = (RRDCALC *) avl_search_lock(&(host)->alarms_idx_health_log, (avl_t *)rc);
if (rdcmp) {
rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_health_log, (avl_t *)rc);
if (!rdcmp) {
error("Cannot remove the health alarm index from health_log");
}
}
rdcmp = (RRDCALC *) avl_search_lock(&(host)->alarms_idx_name, (avl_t *)rc);
if (rdcmp) {
rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_name, (avl_t *)rc);
if (!rdcmp) {
error("Cannot remove the health alarm index from idx_name");
}
}
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->host_alarms, rc, prev, next);
rrdcalc_free(rc);
}
void rrdcalc_foreach_unlink_and_free(RRDHOST *host, RRDCALC *rc) {
if(unlikely(rc == host->alarms_with_foreach))
host->alarms_with_foreach = rc->next;
else {
RRDCALC *t;
for(t = host->alarms_with_foreach; t && t->next != rc; t = t->next) ;
if(t) {
t->next = rc->next;
rc->next = NULL;
}
else
error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
}
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->alarms_with_foreach, rc, prev, next);
rrdcalc_free(rc);
}
@ -736,11 +674,11 @@ static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) {
if(!rrdlabels_match_simple_pattern_parsed(host->host_labels, rc->host_labels_pattern, '=')) {
info("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
rc->name,
host->hostname,
rc->host_labels);
rrdcalc_name(rc),
rrdhost_hostname(host),
rrdcalc_host_labels(rc));
if(host->alarms == alarms)
if(host->host_alarms == alarms)
rrdcalc_unlink_and_free(host, rc);
else
rrdcalc_foreach_unlink_and_free(host, rc);
@ -750,7 +688,7 @@ static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) {
}
void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host) {
rrdcalc_labels_unlink_alarm_loop(host, host->alarms);
rrdcalc_labels_unlink_alarm_loop(host, host->host_alarms);
rrdcalc_labels_unlink_alarm_loop(host, host->alarms_with_foreach);
}
@ -773,59 +711,3 @@ void rrdcalc_labels_unlink() {
rrd_unlock();
}
// ----------------------------------------------------------------------------
// Alarm
/**
* Alarm is repeating
*
* Is this alarm repeating ?
*
* @param host The structure that has the binary tree
* @param alarm_id the id of the alarm to search
*
* @return It returns 1 case it is repeating and 0 otherwise
*/
int alarm_isrepeating(RRDHOST *host, uint32_t alarm_id) {
RRDCALC findme;
findme.id = alarm_id;
RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_health_log, (avl_t *)&findme);
if (!rc) {
return 0;
}
return rrdcalc_isrepeating(rc);
}
/**
* Entry is repeating
*
* Check whether the id of alarm entry is yet present in the host structure
*
* @param host The structure that has the binary tree
* @param ae the alarm entry
*
* @return It returns 1 case it is repeating and 0 otherwise
*/
int alarm_entry_isrepeating(RRDHOST *host, ALARM_ENTRY *ae) {
return alarm_isrepeating(host, ae->alarm_id);
}
/**
* Max last repeat
*
* Check the maximum last_repeat for the alarms associated a host
*
* @param host The structure that has the binary tree
*
* @return It returns 1 case it is repeating and 0 otherwise
*/
RRDCALC *alarm_max_last_repeat(RRDHOST *host, char *alarm_name,uint32_t hash) {
RRDCALC findme;
findme.name = alarm_name;
findme.hash = hash;
RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_name, (avl_t *)&findme);
return rc;
}

142
database/rrdcalc.h
View File

@ -33,33 +33,32 @@
struct rrdcalc {
avl_t avl; // the index, with key the id - this has to be first!
uint32_t id; // the unique id of this alarm
uint32_t next_event_id; // the next event id that will be used for this alarm
char *name; // the name of this alarm
uint32_t hash; // the hash of the alarm name
uuid_t config_hash_id; // a predictable hash_id based on specific alert configuration
char *exec; // the command to execute when this alarm switches state
char *recipient; // the recipient of the alarm (the first parameter to exec)
STRING *name; // the name of this alarm
STRING *chart; // the chart id this should be linked to
char *classification; // the class that this alarm belongs
char *component; // the component that this alarm refers to
char *type; // type of the alarm
STRING *exec; // the command to execute when this alarm switches state
STRING *recipient; // the recipient of the alarm (the first parameter to exec)
char *chart; // the chart id this should be linked to
uint32_t hash_chart;
STRING *classification; // the class that this alarm belongs
STRING *component; // the component that this alarm refers to
STRING *type; // type of the alarm
char *plugin_match; //the plugin name that should be linked to
STRING *plugin_match; // the plugin name that should be linked to
SIMPLE_PATTERN *plugin_pattern;
char *module_match; //the module name that should be linked to
STRING *module_match; // the module name that should be linked to
SIMPLE_PATTERN *module_pattern;
char *source; // the source of this alarm
char *units; // the units of the alarm
char *original_info; // the original info field before any variable replacement
char *info; // a short description of the alarm
STRING *source; // the source of this alarm
STRING *units; // the units of the alarm
STRING *original_info; // the original info field before any variable replacement
STRING *info; // a short description of the alarm
int update_every; // update frequency for the alarm
@ -70,8 +69,8 @@ struct rrdcalc {
// ------------------------------------------------------------------------
// database lookup settings
char *dimensions; // the chart dimensions
char *foreachdim; // the group of dimensions that the `foreach` will be applied.
STRING *dimensions; // the chart dimensions
STRING *foreachdim; // the group of dimensions that the `foreach` will be applied.
SIMPLE_PATTERN *spdim; // used if and only if there is a simple pattern for the chart.
int foreachcounter; // the number of alarms created with foreachdim, this also works as an id of the
// children
@ -99,29 +98,29 @@ struct rrdcalc {
// ------------------------------------------------------------------------
// notification repeat settings
uint32_t warn_repeat_every; // interval between repeating warning notifications
uint32_t crit_repeat_every; // interval between repeating critical notifications
uint32_t warn_repeat_every; // interval between repeating warning notifications
uint32_t crit_repeat_every; // interval between repeating critical notifications
// ------------------------------------------------------------------------
// Labels settings
char *host_labels; // the label read from an alarm file
STRING *host_labels; // the label read from an alarm file
SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
// ------------------------------------------------------------------------
// runtime information
RRDCALC_STATUS old_status; // the old status of the alarm
RRDCALC_STATUS old_status; // the old status of the alarm
RRDCALC_STATUS status; // the current status of the alarm
NETDATA_DOUBLE value; // the current value of the alarm
NETDATA_DOUBLE old_value; // the previous value of the alarm
NETDATA_DOUBLE value; // the current value of the alarm
NETDATA_DOUBLE old_value; // the previous value of the alarm
uint32_t rrdcalc_flags; // check RRDCALC_FLAG_*
time_t last_updated; // the last update timestamp of the alarm
time_t next_update; // the next update timestamp of the alarm
time_t last_status_change; // the timestamp of the last time this alarm changed status
time_t last_repeat; // the last time the alarm got repeated
time_t last_repeat; // the last time the alarm got repeated
uint32_t times_repeat; // number of times the alarm got repeated
time_t db_after; // the first timestamp evaluated by the db lookup
@ -150,49 +149,70 @@ struct rrdcalc {
struct rrdcalc *rrdset_prev;
struct rrdcalc *next;
struct rrdcalc *prev;
};
struct alert_config {
char *alarm;
char *template_key;
char *os;
char *host;
char *on;
char *families;
char *plugin;
char *module;
char *charts;
char *lookup;
char *calc;
char *warn;
char *crit;
char *every;
char *green;
char *red;
char *exec;
char *to;
char *units;
char *info;
char *classification;
char *component;
char *type;
char *delay;
char *options;
char *repeat;
char *host_labels;
#define rrdcalc_name(rc) string2str((rc)->name)
#define rrdcalc_chart_name(rc) string2str((rc)->chart)
#define rrdcalc_exec(rc) string2str((rc)->exec)
#define rrdcalc_recipient(rc) string2str((rc)->recipient)
#define rrdcalc_classification(rc) string2str((rc)->classification)
#define rrdcalc_component(rc) string2str((rc)->component)
#define rrdcalc_type(rc) string2str((rc)->type)
#define rrdcalc_plugin_match(rc) string2str((rc)->plugin_match)
#define rrdcalc_module_match(rc) string2str((rc)->module_match)
#define rrdcalc_source(rc) string2str((rc)->source)
#define rrdcalc_units(rc) string2str((rc)->units)
#define rrdcalc_original_info(rc) string2str((rc)->original_info)
#define rrdcalc_info(rc) string2str((rc)->info)
#define rrdcalc_dimensions(rc) string2str((rc)->dimensions)
#define rrdcalc_foreachdim(rc) string2str((rc)->foreachdim)
#define rrdcalc_host_labels(rc) string2str((rc)->host_labels)
#define foreach_rrdcalc_in_rrdset(st, rc) \
DOUBLE_LINKED_LIST_FOREACH_FORWARD((st)->alarms, rc, rrdset_prev, rrdset_next)
#define foreach_rrdcalc_in_rrdhost(host, rc) \
DOUBLE_LINKED_LIST_FOREACH_FORWARD((host)->host_alarms, rc, prev, next)
struct alert_config {
STRING *alarm;
STRING *template_key;
STRING *os;
STRING *host;
STRING *on;
STRING *families;
STRING *plugin;
STRING *module;
STRING *charts;
STRING *lookup;
STRING *calc;
STRING *warn;
STRING *crit;
STRING *every;
STRING *green;
STRING *red;
STRING *exec;
STRING *to;
STRING *units;
STRING *info;
STRING *classification;
STRING *component;
STRING *type;
STRING *delay;
STRING *options;
STRING *repeat;
STRING *host_labels;
STRING *p_db_lookup_dimensions;
STRING *p_db_lookup_method;
char *p_db_lookup_dimensions;
char *p_db_lookup_method;
uint32_t p_db_lookup_options;
int32_t p_db_lookup_after;
int32_t p_db_lookup_before;
int32_t p_update_every;
};
extern int alarm_isrepeating(RRDHOST *host, uint32_t alarm_id);
extern int alarm_entry_isrepeating(RRDHOST *host, ALARM_ENTRY *ae);
extern RRDCALC *alarm_max_last_repeat(RRDHOST *host, char *alarm_name, uint32_t hash);
#define RRDCALC_HAS_DB_LOOKUP(rc) ((rc)->after)
extern void rrdsetcalc_link_matching(RRDSET *st);
@ -204,13 +224,13 @@ extern const char *rrdcalc_status2string(RRDCALC_STATUS status);
extern void rrdcalc_free(RRDCALC *rc);
extern void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc);
extern int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name);
extern uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id);
extern int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name);
extern uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id);
extern RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart);
extern RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const char *name, const char *dimension);
extern void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc);
extern void dimension_remove_pipe_comma(char *str);
extern char *alarm_name_with_dim(char *name, size_t namelen, const char *dim, size_t dimlen);
extern char *alarm_name_with_dim(const char *name, size_t namelen, const char *dim, size_t dimlen);
extern void rrdcalc_update_rrdlabels(RRDSET *st);
extern void rrdcalc_labels_unlink();

73
database/rrdcalctemplate.c
View File

@ -12,30 +12,30 @@
* @param st is the chart where the alarm will be attached.
*/
void rrdcalctemplate_check_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) {
if(rt->hash_context != st->hash_context || strcmp(rt->context, st->context) != 0)
if(rt->context != st->context)
return;
if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, st->name))
if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, rrdset_name(st)))
return;
if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, st->family))
if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, rrdset_family(st)))
return;
if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, st->module_name))
if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, rrdset_module_name(st)))
return;
if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, st->plugin_name))
if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, rrdset_plugin_name(st)))
return;
if(host->host_labels && rt->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(host->host_labels, rt->host_labels_pattern, '='))
return;
RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id);
RRDCALC *rc = rrdcalc_create_from_template(host, rt, rrdset_id(st));
if (unlikely(!rc))
info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, host->hostname);
info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rrdcalctemplate_name(rt), rrdset_id(st), rrdhost_hostname(host));
#ifdef NETDATA_INTERNAL_CHECKS
else if (rc->rrdset != st && !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late
error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart ? rc->chart : "NOCHART", rc->name, st->id);
else if (rc->rrdset != st && !rc->foreachdim) //When we have a template with foreachdim, the child will be added to the index late
error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st));
#endif
}
@ -43,10 +43,7 @@ void rrdcalctemplate_link_matching(RRDSET *st) {
RRDHOST *host = st->rrdhost;
RRDCALCTEMPLATE *rt;
for(rt = host->templates; rt ; rt = rt->next)
rrdcalctemplate_check_conditions_and_link(rt, st, host);
for(rt = host->alarms_template_with_foreach; rt ; rt = rt->next)
foreach_rrdcalctemplate_in_rrdhost(host, rt)
rrdcalctemplate_check_conditions_and_link(rt, st, host);
}
@ -57,31 +54,31 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) {
expression_free(rt->warning);
expression_free(rt->critical);
freez(rt->family_match);
string_freez(rt->family_match);
simple_pattern_free(rt->family_pattern);
freez(rt->plugin_match);
string_freez(rt->plugin_match);
simple_pattern_free(rt->plugin_pattern);
freez(rt->module_match);
string_freez(rt->module_match);
simple_pattern_free(rt->module_pattern);
freez(rt->charts_match);
string_freez(rt->charts_match);
simple_pattern_free(rt->charts_pattern);
freez(rt->name);
freez(rt->exec);
freez(rt->recipient);
freez(rt->classification);
freez(rt->component);
freez(rt->type);
freez(rt->context);
freez(rt->source);
freez(rt->units);
freez(rt->info);
freez(rt->dimensions);
freez(rt->foreachdim);
freez(rt->host_labels);
string_freez(rt->name);
string_freez(rt->exec);
string_freez(rt->recipient);
string_freez(rt->classification);
string_freez(rt->component);
string_freez(rt->type);
string_freez(rt->context);
string_freez(rt->source);
string_freez(rt->units);
string_freez(rt->info);
string_freez(rt->dimensions);
string_freez(rt->foreachdim);
string_freez(rt->host_labels);
simple_pattern_free(rt->spdim);
simple_pattern_free(rt->host_labels_pattern);
freez(rt);
@ -90,21 +87,9 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) {
inline void rrdcalctemplate_unlink_and_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
if(unlikely(!rt)) return;
debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname);
debug(D_HEALTH, "Health removing template '%s' of host '%s'", rrdcalctemplate_name(rt), rrdhost_hostname(host));
if(host->templates == rt) {
host->templates = rt->next;
}
else {
RRDCALCTEMPLATE *t;
for (t = host->templates; t && t->next != rt; t = t->next ) ;
if(t) {
t->next = rt->next;
rt->next = NULL;
}
else
error("Cannot find RRDCALCTEMPLATE '%s' linked in host '%s'", rt->name, host->hostname);
}
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->alarms_templates, rt, prev, next);
rrdcalctemplate_free(rt);
}

58
database/rrdcalctemplate.h
View File

@ -9,35 +9,34 @@
// these are to be applied to charts found dynamically
// based on their context.
struct rrdcalctemplate {
char *name;
uint32_t hash_name;
uuid_t config_hash_id;
char *exec;
char *recipient;
STRING *name;
char *classification;
char *component;
char *type;
STRING *exec;
STRING *recipient;
char *context;
uint32_t hash_context;
STRING *classification;
STRING *component;
STRING *type;
char *family_match;
STRING *context;
STRING *family_match;
SIMPLE_PATTERN *family_pattern;
char *plugin_match;
STRING *plugin_match;
SIMPLE_PATTERN *plugin_pattern;
char *module_match;
STRING *module_match;
SIMPLE_PATTERN *module_pattern;
char *charts_match;
STRING *charts_match;
SIMPLE_PATTERN *charts_pattern;
char *source; // the source of this alarm
char *units; // the units of the alarm
char *info; // a short description of the alarm
STRING *source; // the source of this alarm
STRING *units; // the units of the alarm
STRING *info; // a short description of the alarm
int update_every; // update frequency for the alarm
@ -48,8 +47,8 @@ struct rrdcalctemplate {
// ------------------------------------------------------------------------
// database lookup settings
char *dimensions; // the chart dimensions
char *foreachdim; // the group of dimensions that the lookup will be applied.
STRING *dimensions; // the chart dimensions
STRING *foreachdim; // the group of dimensions that the lookup will be applied.
SIMPLE_PATTERN *spdim; // used if and only if there is a simple pattern for the chart.
int foreachcounter; // the number of alarms created with foreachdim, this also works as an id of the
// children
@ -74,7 +73,7 @@ struct rrdcalctemplate {
// ------------------------------------------------------------------------
// Labels settings
char *host_labels; // the label read from an alarm file
STRING *host_labels; // the label read from an alarm file
SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
// ------------------------------------------------------------------------
@ -85,8 +84,29 @@ struct rrdcalctemplate {
EVAL_EXPRESSION *critical;
struct rrdcalctemplate *next;
struct rrdcalctemplate *prev;
};
#define foreach_rrdcalctemplate_in_rrdhost(host, rt) \
DOUBLE_LINKED_LIST_FOREACH_FORWARD((host)->alarms_templates, rt, prev, next)
#define rrdcalctemplate_name(rt) string2str((rt)->name)
#define rrdcalctemplate_exec(rt) string2str((rt)->exec)
#define rrdcalctemplate_recipient(rt) string2str((rt)->recipient)
#define rrdcalctemplate_classification(rt) string2str((rt)->classification)
#define rrdcalctemplate_component(rt) string2str((rt)->component)
#define rrdcalctemplate_type(rt) string2str((rt)->type)
#define rrdcalctemplate_family_match(rt) string2str((rt)->family_match)
#define rrdcalctemplate_plugin_match(rt) string2str((rt)->plugin_match)
#define rrdcalctemplate_module_match(rt) string2str((rt)->module_match)
#define rrdcalctemplate_charts_match(rt) string2str((rt)->charts_match)
#define rrdcalctemplate_units(rt) string2str((rt)->units)
#define rrdcalctemplate_info(rt) string2str((rt)->info)
#define rrdcalctemplate_source(rt) string2str((rt)->source)
#define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions)
#define rrdcalctemplate_foreachdim(rt) string2str((rt)->foreachdim)
#define rrdcalctemplate_host_labels(rt) string2str((rt)->host_labels)
#define RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) ((rt)->after)
extern void rrdcalctemplate_link_matching(RRDSET *st);

450
database/rrdcontext.c
View File

@ -14,7 +14,6 @@ int rrdcontext_enabled = CONFIG_BOOLEAN_YES;
#define RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY 10
// #define LOG_TRANSITIONS 1
// #define LOG_RRDINSTANCES 1
typedef enum {
RRD_FLAG_NONE = 0,
@ -49,6 +48,9 @@ typedef enum {
RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 28), // this context changed because of a db rotation
RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 29), // this context is not used anymore
RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS = (1 << 30), // this context is not used anymore
// DO NOT ADD (1 << 31) or bigger!
// runtime error: left shift of 1 by 31 places cannot be represented in type 'int'
} RRD_FLAGS;
#define RRD_FLAG_ALL_UPDATE_REASONS ( \
@ -85,10 +87,6 @@ typedef enum {
#define RRD_FLAGS_PREVENTING_DELETIONS ( \
RRD_FLAG_QUEUED \
|RRD_FLAG_COLLECTED \
|RRD_FLAG_UPDATE_REASON_LOAD_SQL \
|RRD_FLAG_UPDATE_REASON_NEW_OBJECT \
|RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT \
|RRD_FLAG_UPDATE_REASON_CHANGED_LINKING \
)
#define rrd_flag_set_updated(obj, reason) (obj)->flags |= (RRD_FLAG_UPDATED | (reason))
@ -295,6 +293,8 @@ static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reaso
#define rrdcontext_version_hash(host) rrdcontext_version_hash_with_callback(host, NULL, false, NULL)
static uint64_t rrdcontext_version_hash_with_callback(RRDHOST *host, void (*callback)(RRDCONTEXT *, bool, void *), bool snapshot, void *bundle);
static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker);
static void rrdcontext_garbage_collect(void);
void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc);
#define rrdcontext_lock(rc) netdata_mutex_lock(&((rc)->mutex))
@ -303,9 +303,9 @@ void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc);
// ----------------------------------------------------------------------------
// Updates triggers
static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate);
static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate);
static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force);
static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate, const char *function);
static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate, const char *function);
static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force, const char *function);
// ----------------------------------------------------------------------------
// visualizing flags
@ -354,109 +354,46 @@ static void rrd_reasons_to_buffer(RRD_FLAGS flags, BUFFER *wb) {
// logging of all data collected
#ifdef LOG_TRANSITIONS
static void log_transition(STRING *metric, STRING *instance, STRING *context, RRD_FLAGS flags, const char *msg) {
static void log_transition(RRDMETRIC *rm, RRDINSTANCE *ri, RRDCONTEXT *rc, const char *function) {
BUFFER *wb = buffer_create(1000);
const char *triggered_on = "triggered on ";
buffer_sprintf(wb, "RRD TRANSITION: context '%s'", string2str(context));
buffer_sprintf(wb, "RRD TRANSITION: %s() ", function);
if(instance)
buffer_sprintf(wb, ", instance '%s'", string2str(instance));
if(rm) {
buffer_sprintf(wb, "%smetric '%s' of ", triggered_on, string2str(rm->id));
triggered_on = "";
}
if(metric)
buffer_sprintf(wb, ", metric '%s'", string2str(metric));
if(ri) {
buffer_sprintf(wb, "%sinstance '%s' of ", triggered_on, string2str(ri->id));
triggered_on = "";
}
buffer_sprintf(wb, ", triggered by %s: ", msg);
buffer_sprintf(wb, "%scontext '%s' ", triggered_on, string2str(rc->id));
RRD_FLAGS flags = rc->flags;
const char *we_are = "context";
if(ri) {
flags = ri->flags;
we_are = "instance";
}
if(rm) {
flags = rm->flags;
we_are = "metric";
}
buffer_sprintf(wb, "%s flags: ", we_are);
rrd_flags_to_buffer(flags, wb);
buffer_strcat(wb, ", reasons: ");
buffer_strcat(wb, ", having reasons: ");
rrd_reasons_to_buffer(flags, wb);
internal_error(true, "%s", buffer_tostring(wb));
buffer_free(wb);
}
#else
#define log_transition(metric, instance, context, flags, msg) debug_dummy()
#endif
#ifdef LOG_RRDINSTANCES
static void rrdinstance_log(RRDINSTANCE *ri, const char *msg) {
char uuid[UUID_STR_LEN];
uuid_unparse(ri->uuid, uuid);
BUFFER *wb = buffer_create(1000);
buffer_sprintf(wb,
"RRDINSTANCE: %s id '%s' (host '%s'), uuid '%s', name '%s', context '%s', title '%s', units '%s', family '%s', priority %zu, chart type '%s', update every %d, rrdset '%s', flags %s%s%s%s%s%s%s%s, first_time_t %ld, last_time_t %ld",
msg,
string2str(ri->id),
ri->rc->rrdhost->hostname,
uuid,
string2str(ri->name),
string2str(ri->rc->id),
string2str(ri->title),
string2str(ri->units),
string2str(ri->family),
ri->priority,
rrdset_type_name(ri->chart_type),
ri->update_every,
ri->rrdset?ri->rrdset->id:"NONE",
ri->flags & RRD_FLAG_DELETED ?"DELETED ":"",
ri->flags & RRD_FLAG_UPDATED ?"UPDATED ":"",
rrd_flag_is_collected(ri) ?"COLLECTED ":"",
rrd_flag_is_archived(ri) ?"ARCHIVED ":"",
ri->flags & RRD_FLAG_OWNLABELS ?"OWNLABELS ":"",
ri->flags & RRD_FLAG_LIVE_RETENTION ?"LIVE ":"",
ri->flags & RRD_FLAG_QUEUED ?"QUEUED ":"",
ri->flags & RRD_FLAG_DONT_TRIGGER ?"BLOCKED ":"",
ri->first_time_t,
ri->last_time_t
);
buffer_strcat(wb, ", update reasons: { ");
for(int i = 0, added = 0; rrdcontext_reasons[i].name ;i++)
if(ri->flags & rrdcontext_reasons[i].flag) {
if(added) buffer_strcat(wb, ", ");
buffer_strcat(wb, rrdcontext_reasons[i].name);
added++;
}
buffer_strcat(wb, " }");
buffer_strcat(wb, ", labels: { ");
if(ri->rrdlabels) {
if(!rrdlabels_to_buffer(ri->rrdlabels, wb, "", "=", "'", ", ", NULL, NULL, NULL, NULL))
buffer_strcat(wb, "EMPTY }");
else
buffer_strcat(wb, " }");
}
else
buffer_strcat(wb, "NONE }");
buffer_strcat(wb, ", metrics: { ");
if(ri->rrdmetrics) {
RRDMETRIC *v;
int i = 0;
dfe_start_read((DICTIONARY *)ri->rrdmetrics, v) {
buffer_sprintf(wb, "%s%s", i?",":"", v_name);
i++;
}
dfe_done(v);
if(!i)
buffer_strcat(wb, "EMPTY }");
else
buffer_strcat(wb, " }");
}
else
buffer_strcat(wb, "NONE }");
internal_error(true, "%s", buffer_tostring(wb));
buffer_free(wb);
}
#else
#define rrdinstance_log(ir, msg) debug_dummy()
#define log_transition(rm, ri, rc, function) debug_dummy()
#endif
// ----------------------------------------------------------------------------
@ -577,7 +514,7 @@ static void rrdmetric_conflict_callback(const char *id __maybe_unused, void *old
char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
uuid_unparse(rm->uuid, uuid1);
uuid_unparse(rm_new->uuid, uuid2);
internal_error(true, "RRDMETRIC: '%s' is linked to RRDDIM '%s' but they have different UUIDs. RRDMETRIC has '%s', RRDDIM has '%s'", string2str(rm->id), rm->rrddim->id, uuid1, uuid2);
internal_error(true, "RRDMETRIC: '%s' is linked to RRDDIM '%s' but they have different UUIDs. RRDMETRIC has '%s', RRDDIM has '%s'", string2str(rm->id), rrddim_id(rm->rrddim), uuid1, uuid2);
}
if(rm->rrddim != rm_new->rrddim)
@ -616,7 +553,7 @@ static void rrdmetric_conflict_callback(const char *id __maybe_unused, void *old
static void rrdmetric_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) {
RRDMETRIC *rm = value;
rrdmetric_trigger_updates(rm, false, true);
rrdmetric_trigger_updates(rm, false, true, __FUNCTION__);
}
static void rrdmetrics_create(RRDINSTANCE *ri) {
@ -649,8 +586,8 @@ static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) {
if(likely(rm->rrddim))
return false;
if((now_realtime_usec() - rm->created_ut) < 600 * USEC_PER_SEC)
return false;
//if((now_realtime_usec() - rm->created_ut) < 600 * USEC_PER_SEC)
// return false;
rrdmetric_update_retention(rm);
if(rm->first_time_t || rm->last_time_t)
@ -659,38 +596,37 @@ static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) {
return true;
}
static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate) {
static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate, const char *function __maybe_unused) {
if(likely(!force && !(rm->flags & RRD_FLAG_UPDATED))) return;
if(unlikely(rrd_flag_is_collected(rm) && !rm->rrddim))
rrd_flag_set_archived(rm);
// logs and statistics
log_transition(rm, rm->ri, rm->ri->rc, function);
rrdcontext_triggered_update_on_rrdmetric();
if(unlikely((rm->flags & RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD) && rrd_flag_is_collected(rm)))
rrd_flag_set_archived(rm);
if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rm->flags & RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD))
rrd_flag_set_archived(rm);
rrdmetric_update_retention(rm);
if(unlikely(escalate && rm->flags & RRD_FLAG_UPDATED && !(rm->ri->flags & RRD_FLAG_DONT_PROCESS))) {
log_transition(rm->id, rm->ri->id, rm->ri->rc->id, rm->flags, "RRDMETRIC");
rrdinstance_trigger_updates(rm->ri, true, true);
}
if(unlikely(escalate && rm->flags & RRD_FLAG_UPDATED))
rrdinstance_trigger_updates(rm->ri, true, true, __FUNCTION__);
}
static inline void rrdmetric_from_rrddim(RRDDIM *rd) {
if(unlikely(!rd->rrdset))
fatal("RRDMETRIC: rrddim '%s' does not have a rrdset.", rd->id);
fatal("RRDMETRIC: rrddim '%s' does not have a rrdset.", rrddim_id(rd));
if(unlikely(!rd->rrdset->rrdhost))
fatal("RRDMETRIC: rrdset '%s' does not have a rrdhost", rd->rrdset->id);
fatal("RRDMETRIC: rrdset '%s' does not have a rrdhost", rrdset_id(rd->rrdset));
if(unlikely(!rd->rrdset->rrdinstance))
fatal("RRDMETRIC: rrdset '%s' does not have a rrdinstance", rd->rrdset->id);
fatal("RRDMETRIC: rrdset '%s' does not have a rrdinstance", rrdset_id(rd->rrdset));
RRDINSTANCE *ri = rrdinstance_acquired_value(rd->rrdset->rrdinstance);
RRDMETRIC trm = {
.id = string_strdupz(rd->id),
.name = string_strdupz(rd->name),
.id = string_dup(rd->id),
.name = string_dup(rd->name),
.flags = RRD_FLAG_NONE,
.rrddim = rd,
};
@ -707,14 +643,14 @@ static inline void rrdmetric_from_rrddim(RRDDIM *rd) {
#define rrddim_get_rrdmetric(rd) rrddim_get_rrdmetric_with_trace(rd, __FUNCTION__)
static inline RRDMETRIC *rrddim_get_rrdmetric_with_trace(RRDDIM *rd, const char *function) {
if(unlikely(!rd->rrdmetric)) {
error("RRDMETRIC: RRDDIM '%s' is not linked to an RRDMETRIC at %s()", rd->id, function);
error("RRDMETRIC: RRDDIM '%s' is not linked to an RRDMETRIC at %s()", rrddim_id(rd), function);
return NULL;
}
RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdmetric);
if(unlikely(rm->rrddim != rd))
fatal("RRDMETRIC: '%s' is not linked to RRDDIM '%s' at %s()", string2str(rm->id), rd->id, function);
fatal("RRDMETRIC: '%s' is not linked to RRDDIM '%s' at %s()", string2str(rm->id), rrddim_id(rd), function);
return rm;
}
@ -727,7 +663,7 @@ static inline void rrdmetric_rrddim_is_freed(RRDDIM *rd) {
rrd_flag_set_archived(rm);
rm->rrddim = NULL;
rrdmetric_trigger_updates(rm, false, true);
rrdmetric_trigger_updates(rm, false, true, __FUNCTION__);
rrdmetric_release(rd->rrdmetric);
rd->rrdmetric = NULL;
}
@ -736,12 +672,12 @@ static inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) {
RRDMETRIC *rm = rrddim_get_rrdmetric(rd);
if(unlikely(!rm)) return;
if(unlikely(rd->flags & (RRDDIM_FLAG_ARCHIVED | RRDDIM_FLAG_OBSOLETE))) {
if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED|RRDDIM_FLAG_OBSOLETE))) {
if(unlikely(rrd_flag_is_collected(rm)))
rrd_flag_set_archived(rm);
}
rrdmetric_trigger_updates(rm, false, true);
rrdmetric_trigger_updates(rm, false, true, __FUNCTION__);
}
static inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
@ -751,7 +687,7 @@ static inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
if(unlikely(!rrd_flag_is_collected(rm)))
rrd_flag_set_collected(rm);
rrdmetric_trigger_updates(rm, false, true);
rrdmetric_trigger_updates(rm, false, true, __FUNCTION__);
}
// ----------------------------------------------------------------------------
@ -807,7 +743,7 @@ static void rrdinstance_insert_callback(const char *id __maybe_unused, void *val
}
if(ri->rrdset) {
if(unlikely((ri->rrdset->flags & RRDSET_FLAG_HIDDEN) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart)))
if(unlikely((rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart)))
ri->flags |= RRD_FLAG_HIDDEN;
else
ri->flags &= ~RRD_FLAG_HIDDEN;
@ -818,7 +754,6 @@ static void rrdinstance_insert_callback(const char *id __maybe_unused, void *val
ri->flags |= RRD_FLAG_HIDDEN;
rrdmetrics_create(ri);
rrdinstance_log(ri, "INSERT");
// signal the react callback to do the job
rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
@ -829,8 +764,6 @@ static void rrdinstance_delete_callback(const char *id, void *value, void *data)
RRDCONTEXT *rc = data; (void)rc;
RRDINSTANCE *ri = (RRDINSTANCE *)value;
rrdinstance_log(ri, "DELETE");
internal_error(ri->rrdset, "RRDINSTANCE: '%s' is freed but there is a RRDSET linked to it.", string2str(ri->id));
rrdinstance_free(ri);
@ -858,7 +791,7 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o
char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
uuid_unparse(ri->uuid, uuid1);
uuid_unparse(*ri->rrdset->chart_uuid, uuid2);
internal_error(true, "RRDINSTANCE: '%s' is linked to RRDSET '%s' but they have different UUIDs. RRDINSTANCE has '%s', RRDSET has '%s'", string2str(ri->id), ri->rrdset->id, uuid1, uuid2);
internal_error(true, "RRDINSTANCE: '%s' is linked to RRDSET '%s' but they have different UUIDs. RRDINSTANCE has '%s', RRDSET has '%s'", string2str(ri->id), rrdset_id(ri->rrdset), uuid1, uuid2);
}
if(ri->name != ri_new->name) {
@ -920,7 +853,7 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o
}
if(ri->rrdset) {
if(unlikely((ri->rrdset->flags & RRDSET_FLAG_HIDDEN) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart)))
if(unlikely((rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart)))
ri->flags |= RRD_FLAG_HIDDEN;
else
ri->flags &= ~RRD_FLAG_HIDDEN;
@ -934,8 +867,6 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o
if(ri->flags & RRD_FLAG_UPDATED)
ri->flags |= RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT;
rrdinstance_log(ri, "CONFLICT");
// free the new one
rrdinstance_free(ri_new);
@ -945,7 +876,7 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o
static void rrdinstance_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) {
RRDINSTANCE *ri = value;
rrdinstance_trigger_updates(ri, false, true);
rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
}
void rrdinstances_create(RRDCONTEXT *rc) {
@ -993,10 +924,14 @@ static inline bool rrdinstance_should_be_deleted(RRDINSTANCE *ri) {
return true;
}
static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate) {
static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate, const char *function __maybe_unused) {
if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS)) return;
if(unlikely(!force && !(ri->flags & RRD_FLAG_UPDATED))) return;
// logs and stats
log_transition(NULL, ri, ri->rc, function);
rrdcontext_triggered_update_on_rrdinstance();
if(likely(ri->rrdset)) {
if(unlikely(ri->rrdset->priority != ri->priority)) {
ri->priority = ri->rrdset->priority;
@ -1015,10 +950,10 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala
time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
size_t metrics_active = 0, metrics_deleted = 0;
bool live_retention = true, currently_collected = false;
{
if(dictionary_stats_entries(ri->rrdmetrics) > 0) {
RRDMETRIC *rm;
dfe_start_read((DICTIONARY *)ri->rrdmetrics, rm) {
if(!(rm->flags & RRD_FLAG_LIVE_RETENTION))
if(unlikely(!(rm->flags & RRD_FLAG_LIVE_RETENTION)))
live_retention = false;
if (unlikely((rrdmetric_should_be_deleted(rm)))) {
@ -1027,7 +962,7 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala
continue;
}
if(rm->flags & RRD_FLAG_COLLECTED)
if(rm->flags & RRD_FLAG_COLLECTED && rm->first_time_t)
currently_collected = true;
metrics_active++;
@ -1043,9 +978,9 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala
dfe_done(rm);
}
if(live_retention && !(ri->flags & RRD_FLAG_LIVE_RETENTION))
if(unlikely(live_retention && !(ri->flags & RRD_FLAG_LIVE_RETENTION)))
ri->flags |= RRD_FLAG_LIVE_RETENTION;
else if(!live_retention && (ri->flags & RRD_FLAG_LIVE_RETENTION))
else if(unlikely(!live_retention && (ri->flags & RRD_FLAG_LIVE_RETENTION)))
ri->flags &= ~RRD_FLAG_LIVE_RETENTION;
if(unlikely(!metrics_active)) {
@ -1080,7 +1015,7 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala
rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
}
if(unlikely(live_retention))
if(likely(live_retention))
rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
}
else {
@ -1103,18 +1038,16 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escala
}
}
if(unlikely(escalate && ri->flags & RRD_FLAG_UPDATED && !(ri->rc->flags & RRD_FLAG_DONT_PROCESS))) {
log_transition(NULL, ri->id, ri->rc->id, ri->flags, "RRDINSTANCE");
rrdcontext_trigger_updates(ri->rc, true);
}
if(unlikely(escalate && ri->flags & RRD_FLAG_UPDATED))
rrdcontext_trigger_updates(ri->rc, true, __FUNCTION__);
}
static inline void rrdinstance_from_rrdset(RRDSET *st) {
RRDCONTEXT trc = {
.id = string_strdupz(st->context),
.title = string_strdupz(st->title),
.units = string_strdupz(st->units),
.family = string_strdupz(st->family),
.id = string_dup(st->context),
.title = string_dup(st->title),
.units = string_dup(st->units),
.family = string_dup(st->family),
.priority = st->priority,
.chart_type = st->chart_type,
.flags = RRD_FLAG_NONE,
@ -1125,11 +1058,11 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
RRDINSTANCE tri = {
.id = string_strdupz(st->id),
.name = string_strdupz(st->name),
.units = string_strdupz(st->units),
.family = string_strdupz(st->family),
.title = string_strdupz(st->title),
.id = string_dup(st->id),
.name = string_dup(st->name),
.units = string_dup(st->units),
.family = string_dup(st->family),
.title = string_dup(st->title),
.chart_type = st->chart_type,
.priority = st->priority,
.update_every = st->update_every,
@ -1192,7 +1125,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
ri_old->flags &= ~RRD_FLAG_DONT_PROCESS;
rc_old->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdinstance_trigger_updates(ri_old, true, true);
rrdinstance_trigger_updates(ri_old, true, true, __FUNCTION__);
ri_old->flags |= RRD_FLAG_DONT_PROCESS;
rrdinstance_release(ria_old);
@ -1205,10 +1138,10 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
rc_old->first_time_t = 0;
rc_old->last_time_t = 0;
rrdcontext_unlock(rc_old);
rrdcontext_trigger_updates(rc_old, true);
rrdcontext_trigger_updates(rc_old, true, __FUNCTION__);
}
else
rrdcontext_trigger_updates(rc_old, true);
rrdcontext_trigger_updates(rc_old, true, __FUNCTION__);
*/
rrdcontext_release(rca_old);
@ -1223,14 +1156,14 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
#define rrdset_get_rrdinstance(st) rrdset_get_rrdinstance_with_trace(st, __FUNCTION__);
static inline RRDINSTANCE *rrdset_get_rrdinstance_with_trace(RRDSET *st, const char *function) {
if(unlikely(!st->rrdinstance)) {
error("RRDINSTANCE: RRDSET '%s' is not linked to an RRDINSTANCE at %s()", st->id, function);
error("RRDINSTANCE: RRDSET '%s' is not linked to an RRDINSTANCE at %s()", rrdset_id(st), function);
return NULL;
}
RRDINSTANCE *ri = rrdinstance_acquired_value(st->rrdinstance);
if(unlikely(ri->rrdset != st))
fatal("RRDINSTANCE: '%s' is not linked to RRDSET '%s' at %s()", string2str(ri->id), st->id, function);
fatal("RRDINSTANCE: '%s' is not linked to RRDSET '%s' at %s()", string2str(ri->id), rrdset_id(st), function);
return ri;
}
@ -1250,7 +1183,7 @@ static inline void rrdinstance_rrdset_is_freed(RRDSET *st) {
ri->rrdset = NULL;
ri->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdinstance_trigger_updates(ri, false, true);
rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
ri->flags |= RRD_FLAG_DONT_PROCESS;
rrdinstance_release(st->rrdinstance);
@ -1267,26 +1200,26 @@ static inline void rrdinstance_updated_rrdset_name(RRDSET *st) {
RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
if(unlikely(!ri)) return;
STRING *old = ri->name;
ri->name = string_strdupz(st->name);
if(st->name != ri->name) {
STRING *old = ri->name;
ri->name = string_dup(st->name);
string_freez(old);
if(ri->name != old)
rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_NAME);
string_freez(old);
rrdinstance_trigger_updates(ri, false, true);
rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
}
}
static inline void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, RRDSET *st) {
if(unlikely(st->flags & (RRDSET_FLAG_ARCHIVED | RRDSET_FLAG_OBSOLETE)))
rrd_flag_set_archived(ri);
if(unlikely(ri->rrdset != st))
fatal("RRDCONTEXT: instance '%s' is not linked to chart '%s' on host '%s'",
string2str(ri->id), rrdset_id(st), rrdhost_hostname(st->rrdhost));
if(unlikely((st->flags & RRDSET_FLAG_HIDDEN) && !(ri->flags & RRD_FLAG_HIDDEN))) {
if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)) && !(ri->flags & RRD_FLAG_HIDDEN))) {
ri->flags |= RRD_FLAG_HIDDEN;
rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);
}
else if(unlikely(!(st->flags & RRDSET_FLAG_HIDDEN) && (ri->flags & RRD_FLAG_HIDDEN))) {
else if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)) && (ri->flags & RRD_FLAG_HIDDEN))) {
ri->flags &= ~RRD_FLAG_HIDDEN;
rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);
}
@ -1296,10 +1229,13 @@ static inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {
RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
if(unlikely(!ri)) return;
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED|RRDSET_FLAG_OBSOLETE)))
rrd_flag_set_archived(ri);
rrdinstance_updated_rrdset_flags_no_action(ri, st);
ri->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdinstance_trigger_updates(ri, false, true);
rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
ri->flags |= RRD_FLAG_DONT_PROCESS;
}
@ -1309,13 +1245,16 @@ static inline void rrdinstance_collected_rrdset(RRDSET *st) {
rrdinstance_updated_rrdset_flags_no_action(ri, st);
if(unlikely(!rrd_flag_is_collected(ri)))
rrd_flag_set_collected(ri);
if(dictionary_stats_entries(ri->rrdmetrics) > 0) {
if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS))
ri->flags &= ~RRD_FLAG_DONT_PROCESS;
if(unlikely(!rrd_flag_is_collected(ri)))
rrd_flag_set_collected(ri);
rrdinstance_trigger_updates(ri, false, true);
if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS))
ri->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
}
}
// ----------------------------------------------------------------------------
@ -1595,7 +1534,7 @@ static void rrdcontext_conflict_callback(const char *id, void *oldv, void *newv,
static void rrdcontext_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) {
RRDCONTEXT *rc = (RRDCONTEXT *)value;
rrdcontext_trigger_updates(rc, false);
rrdcontext_trigger_updates(rc, false, __FUNCTION__);
}
void rrdhost_create_rrdcontexts(RRDHOST *host) {
@ -1649,17 +1588,21 @@ static inline bool rrdcontext_should_be_deleted(RRDCONTEXT *rc) {
return true;
}
static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force) {
static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force, const char *function __maybe_unused) {
if(unlikely(rc->flags & RRD_FLAG_DONT_PROCESS)) return;
if(unlikely(!force && !(rc->flags & RRD_FLAG_UPDATED))) return;
// logs and stats
log_transition(NULL, NULL, rc, function);
rrdcontext_triggered_update_on_rrdcontext();
rrdcontext_lock(rc);
size_t min_priority = LONG_MAX;
time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
size_t instances_active = 0, instances_deleted = 0;
bool live_retention = true, currently_collected = false, hidden = true;
{
if(dictionary_stats_entries(rc->rrdinstances) > 0) {
RRDINSTANCE *ri;
dfe_start_read(rc->rrdinstances, ri) {
if(likely(!(ri->flags & RRD_FLAG_HIDDEN)))
@ -1674,7 +1617,7 @@ static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force) {
continue;
}
if(ri->flags & RRD_FLAG_COLLECTED)
if(ri->flags & RRD_FLAG_COLLECTED && ri->first_time_t)
currently_collected = true;
internal_error(rc->units != ri->units,
@ -1768,8 +1711,6 @@ static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force) {
}
if(unlikely(rc->flags & RRD_FLAG_UPDATED)) {
log_transition(NULL, NULL, rc->id, rc->flags, "RRDCONTEXT");
if(check_if_cloud_version_changed_unsafe(rc, false)) {
rc->version = rrdcontext_get_next_version(rc);
@ -1945,7 +1886,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) {
if(rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
info("RRDCONTEXT: received checkpoint command for claim id '%s', node id '%s', while node '%s' has an active context streaming.",
cmd->claim_id, cmd->node_id, host->hostname);
cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
// disable it temporarily, so that our worker will not attempt to send messages in parallel
rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
@ -1955,7 +1896,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) {
if(cmd->version_hash != our_version_hash) {
error("RRDCONTEXT: received version hash %"PRIu64" for host '%s', does not match our version hash %"PRIu64". Sending snapshot of all contexts.",
cmd->version_hash, host->hostname, our_version_hash);
cmd->version_hash, rrdhost_hostname(host), our_version_hash);
#ifdef ENABLE_ACLK
// prepare the snapshot
@ -1977,11 +1918,11 @@ void rrdcontext_hub_checkpoint_command(void *ptr) {
#endif
}
internal_error(true, "RRDCONTEXT: host '%s' enabling streaming of contexts", host->hostname);
internal_error(true, "RRDCONTEXT: host '%s' enabling streaming of contexts", rrdhost_hostname(host));
rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
char node_str[UUID_STR_LEN];
uuid_unparse_lower(*host->node_id, node_str);
log_access("ACLK REQ [%s (%s)]: STREAM CONTEXTS ENABLED", node_str, host->hostname);
log_access("ACLK REQ [%s (%s)]: STREAM CONTEXTS ENABLED", node_str, rrdhost_hostname(host));
}
void rrdcontext_hub_stop_streaming_command(void *ptr) {
@ -2006,12 +1947,12 @@ void rrdcontext_hub_stop_streaming_command(void *ptr) {
if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
error("RRDCONTEXT: received stop streaming command for claim id '%s', node id '%s', but node '%s' does not have active context streaming. Ignoring command.",
cmd->claim_id, cmd->node_id, host->hostname);
cmd->claim_id, cmd->node_id, rrdhost_hostname(host));
return;
}
internal_error(true, "RRDCONTEXT: host '%s' disabling streaming of contexts", host->hostname);
internal_error(true, "RRDCONTEXT: host '%s' disabling streaming of contexts", rrdhost_hostname(host));
rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS);
}
@ -2378,7 +2319,7 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void
int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions) {
if(!host->rrdctx) {
error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, host->hostname);
error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, rrdhost_hostname(host));
return HTTP_RESP_NOT_FOUND;
}
@ -2418,7 +2359,7 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R
int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions) {
if(!host->rrdctx) {
error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, host->hostname);
error("%s(): request for host '%s' that does not have rrdcontexts initialized.", __FUNCTION__, rrdhost_hostname(host));
return HTTP_RESP_NOT_FOUND;
}
@ -2440,7 +2381,7 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
",\n\t\"machine_guid\": \"%s\""
",\n\t\"node_id\": \"%s\""
",\n\t\"claim_id\": \"%s\""
, host->hostname
, rrdhost_hostname(host)
, host->machine_guid
, node_uuid
, host->aclk_state.claimed_id ? host->aclk_state.claimed_id : ""
@ -2529,7 +2470,7 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri);
ctx_get_label_list(&ri->uuid, rrdinstance_load_clabel, ri);
ri->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdinstance_trigger_updates(ri, true, true);
rrdinstance_trigger_updates(ri, true, true, __FUNCTION__);
// let the instance be in "don't process" mode
// so that we process it once, when it is collected
@ -2568,9 +2509,11 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) {
RRDCONTEXT *rc;
dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
rc->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdcontext_trigger_updates(rc, true);
rrdcontext_trigger_updates(rc, true, __FUNCTION__);
}
dfe_done(rc);
rrdcontext_garbage_collect_single_host(host, false);
}
// ----------------------------------------------------------------------------
@ -2679,19 +2622,18 @@ static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS r
rrd_flag_set_updated(rm, reason);
rm->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdmetric_trigger_updates(rm, true, false);
rrdmetric_trigger_updates(rm, true, false, __FUNCTION__);
}
dfe_done(rm);
ri->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdinstance_trigger_updates(ri, true, false);
rrdinstance_trigger_updates(ri, true, false, __FUNCTION__);
ri->flags |= RRD_FLAG_DONT_PROCESS;
}
dfe_done(ri);
rc->flags &= ~RRD_FLAG_DONT_PROCESS;
rrdcontext_trigger_updates(rc, true);
rrdcontext_trigger_updates(rc, true, __FUNCTION__);
}
static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, int job_id) {
@ -2727,50 +2669,88 @@ void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) {
error("RRDCONTEXT: failed to delete context '%s' version %"PRIu64" from SQL.", rc->hub.id, rc->hub.version);
}
static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker) {
internal_error(true, "RRDCONTEXT: garbage collecting context structures of host '%s'", rrdhost_hostname(host));
RRDCONTEXT *rc;
dfe_start_write((DICTIONARY *)host->rrdctx, rc) {
if(worker) worker_is_busy(WORKER_JOB_CLEANUP);
rrdcontext_lock(rc);
RRDINSTANCE *ri;
dfe_start_write(rc->rrdinstances, ri) {
RRDMETRIC *rm;
dfe_start_write(ri->rrdmetrics, rm) {
if(rrdmetric_should_be_deleted(rm)) {
if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
if(dictionary_del_having_write_lock(ri->rrdmetrics, string2str(rm->id)) != 0)
error("RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
string2str(rm->id),
string2str(ri->id),
string2str(rc->id),
rrdhost_hostname(host));
else
internal_error(
true,
"RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', deleted from rrdmetrics dictionary.",
string2str(rm->id),
string2str(ri->id),
string2str(rc->id),
rrdhost_hostname(host));
}
}
dfe_done(rm);
if(rrdinstance_should_be_deleted(ri)) {
if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
if(dictionary_del_having_write_lock(rc->rrdinstances, string2str(ri->id)) != 0)
error("RRDCONTEXT: instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
string2str(ri->id),
string2str(rc->id),
rrdhost_hostname(host));
else
internal_error(
true,
"RRDCONTEXT: instance '%s' of context '%s' of host '%s', deleted from rrdmetrics dictionary.",
string2str(ri->id),
string2str(rc->id),
rrdhost_hostname(host));
}
}
dfe_done(ri);
if(unlikely(rrdcontext_should_be_deleted(rc))) {
if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
rrdcontext_delete_from_sql_unsafe(rc);
if(dictionary_del_having_write_lock((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0)
error("RRDCONTEXT: context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
string2str(rc->id),
rrdhost_hostname(host));
else
internal_error(
true,
"RRDCONTEXT: context '%s' of host '%s', deleted from rrdmetrics dictionary.",
string2str(rc->id),
rrdhost_hostname(host));
fprintf(stderr, "RRDCONTEXT: deleted context '%s'", string2str(rc->id));
}
// the item is referenced in the dictionary
// so, it is still here to unlock, even if we have deleted it
rrdcontext_unlock(rc);
}
dfe_done(rc);
}
static void rrdcontext_garbage_collect(void) {
rrd_rdlock();
RRDHOST *host;
rrdhost_foreach_read(host) {
RRDCONTEXT *rc;
dfe_start_write((DICTIONARY *)host->rrdctx, rc) {
worker_is_busy(WORKER_JOB_CLEANUP);
rrdcontext_lock(rc);
if(unlikely(rrdcontext_should_be_deleted(rc))) {
worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
rrdcontext_delete_from_sql_unsafe(rc);
if(dictionary_del_having_write_lock((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0)
error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.",
string2str(rc->id), host->hostname);
}
else {
RRDINSTANCE *ri;
dfe_start_write(rc->rrdinstances, ri) {
if(rrdinstance_should_be_deleted(ri)) {
worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
dictionary_del_having_write_lock(rc->rrdinstances, string2str(ri->id));
}
else {
RRDMETRIC *rm;
dfe_start_write(ri->rrdmetrics, rm) {
if(rrdmetric_should_be_deleted(rm)) {
worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
dictionary_del_having_write_lock(ri->rrdmetrics, string2str(rm->id));
}
}
dfe_done(rm);
}
}
dfe_done(ri);
}
// the item is referenced in the dictionary
// so, it is still here to unlock, even if we have deleted it
rrdcontext_unlock(rc);
}
dfe_done(rc);
rrdcontext_garbage_collect_single_host(host, true);
}
rrd_unlock();
}
@ -2895,7 +2875,7 @@ void *rrdcontext_main(void *ptr) {
// delete it from the master dictionary
if(dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0)
error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.",
string2str(id), host->hostname);
string2str(id), rrdhost_hostname(host));
string_freez(id);
}

183
database/rrddim.c
View File

@ -10,48 +10,59 @@
// ----------------------------------------------------------------------------
// RRDDIM index
int rrddim_compare(void* a, void* b) {
if(((RRDDIM *)a)->hash < ((RRDDIM *)b)->hash) return -1;
else if(((RRDDIM *)a)->hash > ((RRDDIM *)b)->hash) return 1;
else return strcmp(((RRDDIM *)a)->id, ((RRDDIM *)b)->id);
static inline void rrddim_index_add(RRDSET *st, RRDDIM *rd) {
if(likely(dictionary_set(st->rrddim_root_index, string2str(rd->id), rd, sizeof(RRDDIM)) == rd)) {
rrddim_flag_set(rd, RRDDIM_FLAG_INDEXED_ID);
}
else {
rrddim_flag_clear(rd, RRDDIM_FLAG_INDEXED_ID);
error("RRDDIM: %s() attempted to index duplicate dimension with key '%s' of chart '%s' of host '%s'", __FUNCTION__, rrddim_id(rd), rrdset_id(st), rrdhost_hostname(st->rrdhost));
}
}
#define rrddim_index_add(st, rd) (RRDDIM *)avl_insert_lock(&((st)->dimensions_index), (avl_t *)(rd))
#define rrddim_index_del(st,rd ) (RRDDIM *)avl_remove_lock(&((st)->dimensions_index), (avl_t *)(rd))
static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id, uint32_t hash) {
RRDDIM tmp = {
.id = id,
.hash = (hash)?hash:simple_hash(id)
};
return (RRDDIM *)avl_search_lock(&(st->dimensions_index), (avl_t *) &tmp);
static inline void rrddim_index_del(RRDSET *st, RRDDIM *rd) {
if(rrddim_flag_check(rd, RRDDIM_FLAG_INDEXED_ID)) {
if (likely(dictionary_del(st->rrddim_root_index, string2str(rd->id)) == 0))
rrddim_flag_clear(rd, RRDDIM_FLAG_INDEXED_ID);
else
error("RRDDIM: %s() attempted to delete non-indexed dimension with key '%s' of chart '%s' of host '%s'", __FUNCTION__, rrddim_id(rd), rrdset_id(st), rrdhost_hostname(st->rrdhost));
}
}
static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id) {
return dictionary_get(st->rrddim_root_index, id);
}
// ----------------------------------------------------------------------------
// RRDDIM - find a dimension
inline RRDDIM *rrddim_find(RRDSET *st, const char *id) {
debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", st->name, id);
debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", rrdset_name(st), id);
return rrddim_index_find(st, id, 0);
return rrddim_index_find(st, id);
}
// This will not return dimensions that are archived
RRDDIM *rrddim_find_active(RRDSET *st, const char *id) {
RRDDIM *rd = rrddim_find(st, id);
if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
return NULL;
return rd;
}
// ----------------------------------------------------------------------------
// RRDDIM rename a dimension
inline int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) {
if(unlikely(!name || !*name || (rd->name && !strcmp(rd->name, name))))
if(unlikely(!name || !*name || !strcmp(rrddim_name(rd), name)))
return 0;
debug(D_RRD_CALLS, "rrddim_set_name() from %s.%s to %s.%s", st->name, rd->name, st->name, name);
debug(D_RRD_CALLS, "rrddim_set_name() from %s.%s to %s.%s", rrdset_name(st), rrddim_name(rd), rrdset_name(st), name);
if (rd->name)
freez((void *) rd->name);
rd->name = strdupz(name);
rd->hash_name = simple_hash(rd->name);
string_freez(rd->name);
rd->name = rrd_string_strdupz(name);
if (!st->state->is_ar_chart)
rrddimvar_rename_all(rd);
@ -68,7 +79,7 @@ inline int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm)
if(unlikely(rd->algorithm == algorithm))
return 0;
debug(D_RRD_CALLS, "Updating algorithm of dimension '%s/%s' from %s to %s", st->id, rd->name, rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm));
debug(D_RRD_CALLS, "Updating algorithm of dimension '%s/%s' from %s to %s", rrdset_id(st), rrddim_name(rd), rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm));
rd->algorithm = algorithm;
rd->exposed = 0;
rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
@ -81,7 +92,7 @@ inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip
if(unlikely(rd->multiplier == multiplier))
return 0;
debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, st->id, rd->name, rd->multiplier, multiplier);
debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, rrdset_id(st), rrddim_name(rd), rd->multiplier, multiplier);
rd->multiplier = multiplier;
rd->exposed = 0;
rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
@ -94,7 +105,7 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
if(unlikely(rd->divisor == divisor))
return 0;
debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, st->id, rd->name, rd->divisor, divisor);
debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, rrdset_id(st), rrddim_name(rd), rd->divisor, divisor);
rd->divisor = divisor;
rd->exposed = 0;
rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
@ -107,31 +118,27 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
// RRDDIM create a dimension
void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) {
RRDCALC *rrdc;
RRDCALC *rc;
for (rrdc = host->alarms_with_foreach; rrdc ; rrdc = rrdc->next) {
if (simple_pattern_matches(rrdc->spdim, rd->id) || simple_pattern_matches(rrdc->spdim, rd->name)) {
if (rrdc->hash_chart == st->hash_name || !strcmp(rrdc->chart, st->name) || !strcmp(rrdc->chart, st->id)) {
char *name = alarm_name_with_dim(rrdc->name, strlen(rrdc->name), rd->name, strlen(rd->name));
if(rrdcalc_exists(host, st->name, name, 0, 0)) {
for (rc = host->alarms_with_foreach; rc; rc = rc->next) {
if (simple_pattern_matches(rc->spdim, rrddim_id(rd)) || simple_pattern_matches(rc->spdim, rrddim_name(rd))) {
if (rc->chart == st->name || rc->chart == st->id) {
char *name = alarm_name_with_dim(rrdcalc_name(rc), string_strlen(rc->name), rrddim_name(rd), string_strlen(rd->name));
if(rrdcalc_exists(host, rrdset_name(st), name)) {
freez(name);
continue;
}
netdata_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
RRDCALC *child = rrdcalc_create_from_rrdcalc(rrdc, host, name, rd->name);
RRDCALC *child = rrdcalc_create_from_rrdcalc(rc, host, name, rrddim_name(rd));
netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);
if (child) {
if (child)
rrdcalc_add_to_host(host, child);
RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl_t *)child);
if (rdcmp != child) {
error("Cannot insert the alarm index ID %s",child->name);
}
}
else {
error("Cannot allocate a new alarm.");
rrdc->foreachcounter--;
rc->foreachcounter--;
}
}
}
@ -157,15 +164,20 @@ time_t calc_dimension_liveness(RRDDIM *rd, time_t now)
}
#endif
RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier,
collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode)
{
RRDDIM *rrddim_add_custom(RRDSET *st
, const char *id
, const char *name
, collected_number multiplier
, collected_number divisor
, RRD_ALGORITHM algorithm
, RRD_MEMORY_MODE memory_mode
) {
RRDHOST *host = st->rrdhost;
rrdset_wrlock(st);
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(rd)) {
debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:"<NONAME>");
debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", rrdset_id(st), name?name:"<NONAME>");
int rc = rrddim_set_name(st, rd, name);
rc += rrddim_set_algorithm(st, rd, algorithm);
@ -192,8 +204,8 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
}
if (unlikely(rc)) {
debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
(void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rrddim_id(rd));
(void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor,
rd->algorithm);
#ifdef ENABLE_ACLK
queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec()));
@ -210,11 +222,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
rd = callocz(1, sizeof(RRDDIM));
rd->id = strdupz(id);
rd->hash = simple_hash(rd->id);
rd->id = string_strdupz(id);
rd->name = (name && *name)?strdupz(name):strdupz(rd->id);
rd->hash_name = simple_hash(rd->name);
rd->name = (name && *name)?rrd_string_strdupz(name):string_dup(rd->id);
rd->algorithm = algorithm;
rd->multiplier = multiplier;
@ -231,7 +241,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
if(memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_SAVE) {
if(!rrddim_memory_load_or_create_map_save(st, rd, memory_mode)) {
info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name, rd->name);
info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st), rrddim_name(rd));
memory_mode = RRD_MEMORY_MODE_RAM;
}
}
@ -242,7 +252,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1);
if(!rd->db) {
info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", st->name, rd->name);
info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));
memory_mode = RRD_MEMORY_MODE_ALLOC;
}
else rd->memsize = entries * sizeof(storage_number);
@ -261,6 +271,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
#ifdef ENABLE_ACLK
rd->aclk_live_status = -1;
#endif
(void) find_dimension_uuid(st, rd, &(rd->metric_uuid));
// initialize the db tiers
@ -284,10 +295,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
}
if(!initialized)
error("Failed to initialize all db tiers for chart '%s', dimension '%s", st->name, rd->name);
error("Failed to initialize all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
if(!rd->tiers[0])
error("Failed to initialize the first db tier for chart '%s', dimension '%s", st->name, rd->name);
error("Failed to initialize the first db tier for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
}
store_active_dimension(&rd->metric_uuid);
@ -303,22 +314,19 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
}
if(!initialized)
error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", st->name, rd->name);
error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
}
// append this dimension
if(!st->dimensions)
st->dimensions = rd;
else {
if(st->dimensions) {
RRDDIM *td = st->dimensions;
if(td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor)) {
if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {
#ifdef NETDATA_INTERNAL_CHECKS
info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").",
rd->name,
st->name,
host->hostname,
rrddim_name(rd),
rrdset_name(st),
rrdhost_hostname(host),
rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(td->algorithm),
rd->multiplier, td->multiplier,
rd->divisor, td->divisor
@ -327,19 +335,18 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS);
}
}
for(; td->next; td = td->next) ;
td->next = rd;
}
// append this dimension
DOUBLE_LINKED_LIST_APPEND_UNSAFE(st->dimensions, rd, prev, next);
if(host->health_enabled && !st->state->is_ar_chart) {
rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT);
rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT);
rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT);
}
if(unlikely(rrddim_index_add(st, rd) != rd))
error("RRDDIM: INTERNAL ERROR: attempt to index duplicate dimension '%s' on chart '%s'", rd->id, st->id);
rrddim_index_add(st, rd);
rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM);
rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS);
@ -360,7 +367,7 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
rrdcontext_removed_rrddim(rd);
ml_delete_dimension(rd);
debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name);
debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
@ -382,24 +389,12 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
}
}
if(rd == st->dimensions)
st->dimensions = rd->next;
else {
RRDDIM *i;
for (i = st->dimensions; i && i->next != rd; i = i->next) ;
if (i && i->next == rd)
i->next = rd->next;
else
error("Request to free dimension '%s.%s' but it is not linked.", st->id, rd->name);
}
rd->next = NULL;
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->dimensions, rd, prev, next);
while(rd->variables)
rrddimvar_free(rd->variables);
if(unlikely(rrddim_index_del(st, rd) != rd))
error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id);
rrddim_index_del(st, rd);
// free(rd->annotations);
//#ifdef ENABLE_ACLK
@ -428,8 +423,8 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
freez(rd->db);
}
freez((void *)rd->id);
freez((void *)rd->name);
string_freez(rd->id);
string_freez(rd->name);
freez(rd);
}
@ -438,13 +433,13 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
// RRDDIM - set dimension options
int rrddim_hide(RRDSET *st, const char *id) {
debug(D_RRD_CALLS, "rrddim_hide() for chart %s, dimension %s", st->name, id);
debug(D_RRD_CALLS, "rrddim_hide() for chart %s, dimension %s", rrdset_name(st), id);
RRDHOST *host = st->rrdhost;
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(!rd)) {
error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, host->hostname);
error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
return 1;
}
if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN))
@ -457,12 +452,12 @@ int rrddim_hide(RRDSET *st, const char *id) {
}
int rrddim_unhide(RRDSET *st, const char *id) {
debug(D_RRD_CALLS, "rrddim_unhide() for chart %s, dimension %s", st->name, id);
debug(D_RRD_CALLS, "rrddim_unhide() for chart %s, dimension %s", rrdset_name(st), id);
RRDHOST *host = st->rrdhost;
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(!rd)) {
error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, host->hostname);
error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
return 1;
}
if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN))
@ -475,10 +470,10 @@ int rrddim_unhide(RRDSET *st, const char *id) {
}
inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
debug(D_RRD_CALLS, "rrddim_is_obsolete() for chart %s, dimension %s", st->name, rd->name);
debug(D_RRD_CALLS, "rrddim_is_obsolete() for chart %s, dimension %s", rrdset_name(st), rrddim_name(rd));
if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))) {
info("Cannot obsolete already archived dimension %s from chart %s", rd->name, st->name);
info("Cannot obsolete already archived dimension %s from chart %s", rrddim_name(rd), rrdset_name(st));
return;
}
rrddim_flag_set(rd, RRDDIM_FLAG_OBSOLETE);
@ -487,7 +482,7 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
}
inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
debug(D_RRD_CALLS, "rrddim_isnot_obsolete() for chart %s, dimension %s", st->name, rd->name);
debug(D_RRD_CALLS, "rrddim_isnot_obsolete() for chart %s, dimension %s", rrdset_name(st), rrddim_name(rd));
rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
rrdcontext_updated_rrddim_flags(rd);
@ -497,9 +492,7 @@ inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
// RRDDIM - collect values for a dimension
inline collected_number rrddim_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM *rd, collected_number value) {
debug(D_RRD_CALLS, "rrddim_set_by_pointer() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
rrdcontext_collected_rrddim(rd);
debug(D_RRD_CALLS, "rrddim_set_by_pointer() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, rrdset_name(st), rrddim_name(rd), value);
now_realtime_timeval(&rd->last_collected_time);
rd->collected_value = value;
@ -519,7 +512,7 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value)
RRDHOST *host = st->rrdhost;
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(!rd)) {
error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, host->hostname);
error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host));
return 0;
}
@ -627,7 +620,7 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO
char filename[FILENAME_MAX + 1];
char fullfilename[FILENAME_MAX + 1];
rrdset_strncpyz_name(filename, rd->id, FILENAME_MAX);
rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX);
snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(fullfilename, size,

119
database/rrddimvar.c
View File

@ -16,64 +16,64 @@ static inline void rrddimvar_free_variables(RRDDIMVAR *rs) {
// CHART VARIABLES FOR THIS DIMENSION
rrdvar_free(host, &st->rrdvar_root_index, rs->var_local_id);
rrdvar_free(host, st->rrdvar_root_index, rs->var_local_id);
rs->var_local_id = NULL;
rrdvar_free(host, &st->rrdvar_root_index, rs->var_local_name);
rrdvar_free(host, st->rrdvar_root_index, rs->var_local_name);
rs->var_local_name = NULL;
// FAMILY VARIABLES FOR THIS DIMENSION
rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_id);
rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_id);
rs->var_family_id = NULL;
rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_name);
rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_name);
rs->var_family_name = NULL;
rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_contextid);
rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_contextid);
rs->var_family_contextid = NULL;
rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_contextname);
rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_contextname);
rs->var_family_contextname = NULL;
// HOST VARIABLES FOR THIS DIMENSION
rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_chartidid);
rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartidid);
rs->var_host_chartidid = NULL;
rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_chartidname);
rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartidname);
rs->var_host_chartidname = NULL;
rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_chartnameid);
rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartnameid);
rs->var_host_chartnameid = NULL;
rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_chartnamename);
rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartnamename);
rs->var_host_chartnamename = NULL;
// KEYS
freez(rs->key_id);
string_freez(rs->key_id);
rs->key_id = NULL;
freez(rs->key_name);
string_freez(rs->key_name);
rs->key_name = NULL;
freez(rs->key_fullidid);
string_freez(rs->key_fullidid);
rs->key_fullidid = NULL;
freez(rs->key_fullidname);
string_freez(rs->key_fullidname);
rs->key_fullidname = NULL;
freez(rs->key_contextid);
string_freez(rs->key_contextid);
rs->key_contextid = NULL;
freez(rs->key_contextname);
string_freez(rs->key_contextname);
rs->key_contextname = NULL;
freez(rs->key_fullnameid);
string_freez(rs->key_fullnameid);
rs->key_fullnameid = NULL;
freez(rs->key_fullnamename);
string_freez(rs->key_fullnamename);
rs->key_fullnamename = NULL;
}
@ -88,29 +88,29 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {
// KEYS
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->id, rs->suffix);
rs->key_id = strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", string2str(rs->prefix), rrddim_id(rd), string2str(rs->suffix));
rs->key_id = string_strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
rs->key_name = strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", string2str(rs->prefix), rrddim_name(rd), string2str(rs->suffix));
rs->key_name = string_strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->key_id);
rs->key_fullidid = strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_id(st), string2str(rs->key_id));
rs->key_fullidid = string_strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->key_name);
rs->key_fullidname = strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_id(st), string2str(rs->key_name));
rs->key_fullidname = string_strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->context, rs->key_id);
rs->key_contextid = strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_context(st), string2str(rs->key_id));
rs->key_contextid = string_strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->context, rs->key_name);
rs->key_contextname = strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_context(st), string2str(rs->key_name));
rs->key_contextname = string_strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->key_id);
rs->key_fullnameid = strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_name(st), string2str(rs->key_id));
rs->key_fullnameid = string_strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->key_name);
rs->key_fullnamename = strdupz(buffer);
snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_name(st), string2str(rs->key_name));
rs->key_fullnamename = string_strdupz(buffer);
// CHART VARIABLES FOR THIS DIMENSION
// -----------------------------------
@ -119,8 +119,8 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {
// - $id
// - $name
rs->var_local_id = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_local_name = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_local_id = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_local_name = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
// FAMILY VARIABLES FOR THIS DIMENSION
// -----------------------------------
@ -131,10 +131,10 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {
// - $chart-context.id
// - $chart-context.name
rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_family_id = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_family_name = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_family_contextid = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_family_contextname = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
// HOST VARIABLES FOR THIS DIMENSION
// -----------------------------------
@ -145,33 +145,32 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {
// - $chart-name.id
// - $chart-name.name
rs->var_host_chartidid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_host_chartidname = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_host_chartnameid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnameid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_host_chartnamename = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnamename, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_host_chartidid = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullidid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_host_chartidname = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullidname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_host_chartnameid = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullnameid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
rs->var_host_chartnamename = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullnamename, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
}
RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options) {
RRDSET *st = rd->rrdset;
(void)st;
debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:"");
debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", rrdset_id(st), rrdset_name(st), rrddim_id(rd), (prefix)?prefix:"", rrddim_name(rd), (suffix)?suffix:"");
if(!prefix) prefix = "";
if(!suffix) suffix = "";
RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR));
rs->prefix = strdupz(prefix);
rs->suffix = strdupz(suffix);
rs->prefix = string_strdupz(prefix);
rs->suffix = string_strdupz(suffix);
rs->type = type;
rs->value = value;
rs->options = options;
rs->rrddim = rd;
rs->next = rd->variables;
rd->variables = rs;
DOUBLE_LINKED_LIST_PREPEND_UNSAFE(rd->variables, rs, prev, next);
rrddimvar_create_variables(rs);
@ -182,7 +181,7 @@ void rrddimvar_rename_all(RRDDIM *rd) {
RRDSET *st = rd->rrdset;
(void)st;
debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", rrdset_id(st), rrdset_name(st), rrddim_id(rd), rrddim_name(rd));
RRDDIMVAR *rs, *next = rd->variables;
while((rs = next)) {
@ -193,25 +192,13 @@ void rrddimvar_rename_all(RRDDIM *rd) {
void rrddimvar_free(RRDDIMVAR *rs) {
RRDDIM *rd = rs->rrddim;
RRDSET *st = rd->rrdset;
debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", st->id, st->name, rd->id, rd->name, rs->prefix, rs->suffix);
debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset), rrddim_id(rd), rrddim_name(rd), string2str(rs->prefix), string2str(rs->suffix));
rrddimvar_free_variables(rs);
if(rd->variables == rs) {
debug(D_VARIABLES, "RRDDIMSET removing first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
rd->variables = rs->next;
}
else {
debug(D_VARIABLES, "RRDDIMSET removing non-first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
RRDDIMVAR *t;
for (t = rd->variables; t && t->next != rs; t = t->next) ;
if(!t) error("RRDDIMVAR '%s' not found in dimension '%s/%s' variables linked list", rs->key_name, st->id, rd->id);
else t->next = rs->next;
}
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rd->variables, rs, prev, next);
freez(rs->prefix);
freez(rs->suffix);
string_freez(rs->prefix);
string_freez(rs->suffix);
freez(rs);
}

21
database/rrddimvar.h
View File

@ -11,17 +11,17 @@
// This means, there will be no speed penalty for using
// these variables
struct rrddimvar {
char *prefix;
char *suffix;
STRING *prefix;
STRING *suffix;
char *key_id; // dimension id
char *key_name; // dimension name
char *key_contextid; // context + dimension id
char *key_contextname; // context + dimension name
char *key_fullidid; // chart type.chart id + dimension id
char *key_fullidname; // chart type.chart id + dimension name
char *key_fullnameid; // chart type.chart name + dimension id
char *key_fullnamename; // chart type.chart name + dimension name
STRING *key_id; // dimension id
STRING *key_name; // dimension name
STRING *key_contextid; // context + dimension id
STRING *key_contextname; // context + dimension name
STRING *key_fullidid; // chart type.chart id + dimension id
STRING *key_fullidname; // chart type.chart id + dimension name
STRING *key_fullnameid; // chart type.chart name + dimension id
STRING *key_fullnamename; // chart type.chart name + dimension name
RRDVAR_TYPE type;
void *value;
@ -44,6 +44,7 @@ struct rrddimvar {
struct rrddim *rrddim;
struct rrddimvar *next;
struct rrddimvar *prev;
};

48
database/rrdfamily.c
View File

@ -6,37 +6,38 @@
// ----------------------------------------------------------------------------
// RRDFAMILY index
int rrdfamily_compare(void *a, void *b) {
if(((RRDFAMILY *)a)->hash_family < ((RRDFAMILY *)b)->hash_family) return -1;
else if(((RRDFAMILY *)a)->hash_family > ((RRDFAMILY *)b)->hash_family) return 1;
else return strcmp(((RRDFAMILY *)a)->family, ((RRDFAMILY *)b)->family);
static inline RRDFAMILY *rrdfamily_index_add(RRDHOST *host, RRDFAMILY *rc) {
return dictionary_set(host->rrdfamily_root_index, string2str(rc->family), rc, sizeof(RRDFAMILY));
}
#define rrdfamily_index_add(host, rc) (RRDFAMILY *)avl_insert_lock(&((host)->rrdfamily_root_index), (avl_t *)(rc))
#define rrdfamily_index_del(host, rc) (RRDFAMILY *)avl_remove_lock(&((host)->rrdfamily_root_index), (avl_t *)(rc))
static RRDFAMILY *rrdfamily_index_find(RRDHOST *host, const char *id, uint32_t hash) {
RRDFAMILY tmp;
tmp.family = id;
tmp.hash_family = (hash)?hash:simple_hash(tmp.family);
return (RRDFAMILY *)avl_search_lock(&(host->rrdfamily_root_index), (avl_t *) &tmp);
static inline RRDFAMILY *rrdfamily_index_del(RRDHOST *host, RRDFAMILY *rc) {
dictionary_del(host->rrdfamily_root_index, string2str(rc->family));
return rc;
}
static inline RRDFAMILY *rrdfamily_index_find(RRDHOST *host, const char *id) {
return dictionary_get(host->rrdfamily_root_index, id);
}
// ----------------------------------------------------------------------------
// RRDFAMILY management
RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id) {
RRDFAMILY *rc = rrdfamily_index_find(host, id, 0);
RRDFAMILY *rc = rrdfamily_index_find(host, id);
if(!rc) {
rc = callocz(1, sizeof(RRDFAMILY));
rc->family = strdupz(id);
rc->hash_family = simple_hash(rc->family);
rc->family = string_strdupz(id);
// initialize the variables index
avl_init_lock(&rc->rrdvar_root_index, rrdvar_compare);
rc->rrdvar_root_index = dictionary_create(
DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE
|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
);
RRDFAMILY *ret = rrdfamily_index_add(host, rc);
if(ret != rc)
error("RRDFAMILY: INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", rc->family, (ret)?ret->family:"NONE");
error("RRDFAMILY: INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", string2str(rc->family), (ret)?string2str(ret->family):"NONE");
}
rc->use_count++;
@ -48,12 +49,13 @@ void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc) {
if(!rc->use_count) {
RRDFAMILY *ret = rrdfamily_index_del(host, rc);
if(ret != rc)
error("RRDFAMILY: INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", rc->family, (ret)?ret->family:"NONE");
error("RRDFAMILY: INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", string2str(rc->family), (ret)?string2str(ret->family):"NONE");
else {
debug(D_RRD_CALLS, "RRDFAMILY: Cleaning up remaining family variables for host '%s', family '%s'", host->hostname, rc->family);
rrdvar_free_remaining_variables(host, &rc->rrdvar_root_index);
debug(D_RRD_CALLS, "RRDFAMILY: Cleaning up remaining family variables for host '%s', family '%s'", rrdhost_hostname(host), string2str(rc->family));
rrdvar_free_remaining_variables(host, rc->rrdvar_root_index);
freez((void *) rc->family);
dictionary_destroy(rc->rrdvar_root_index);
string_freez(rc->family);
freez(rc);
}
}

435
database/rrdhost.c
View File

@ -43,107 +43,150 @@ bool is_storage_engine_shared(STORAGE_INSTANCE *engine) {
// ----------------------------------------------------------------------------
// RRDHOST index
// RRDHOST indexes management
int rrdhost_compare(void* a, void* b) {
if(((RRDHOST *)a)->hash_machine_guid < ((RRDHOST *)b)->hash_machine_guid) return -1;
else if(((RRDHOST *)a)->hash_machine_guid > ((RRDHOST *)b)->hash_machine_guid) return 1;
else return strcmp(((RRDHOST *)a)->machine_guid, ((RRDHOST *)b)->machine_guid);
static DICTIONARY *rrdhost_root_index = NULL;
static DICTIONARY *rrdhost_root_index_hostname = NULL;
static inline void rrdhost_init() {
if(unlikely(!rrdhost_root_index)) {
rrdhost_root_index = dictionary_create(
DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
| DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE
| DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
);
}
if(unlikely(!rrdhost_root_index_hostname)) {
rrdhost_root_index_hostname = dictionary_create(
DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
| DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE
| DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
);
}
}
avl_tree_lock rrdhost_root_index = {
.avl_tree = { NULL, rrdhost_compare },
.rwlock = AVL_LOCK_INITIALIZER
};
// ----------------------------------------------------------------------------
// RRDHOST index by UUID
RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash) {
debug(D_RRDHOST, "Searching in index for host with guid '%s'", guid);
RRDHOST tmp;
strncpyz(tmp.machine_guid, guid, GUID_LEN);
tmp.hash_machine_guid = (hash)?hash:simple_hash(tmp.machine_guid);
return (RRDHOST *)avl_search_lock(&(rrdhost_root_index), (avl_t *) &tmp);
inline long rrdhost_hosts_available(void) {
return dictionary_stats_entries(rrdhost_root_index);
}
RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash) {
inline RRDHOST *rrdhost_find_by_guid(const char *guid) {
return dictionary_get(rrdhost_root_index, guid);
}
static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) {
RRDHOST *ret_machine_guid = dictionary_set(rrdhost_root_index, host->machine_guid, host, sizeof(RRDHOST));
if(ret_machine_guid == host)
rrdhost_flag_set(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID);
else {
rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID);
error("RRDHOST: %s() host with machine guid '%s' is already indexed", __FUNCTION__, host->machine_guid);
}
return host;
}
static void rrdhost_index_del_by_guid(RRDHOST *host) {
if(rrdhost_flag_check(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID)) {
if(dictionary_del(rrdhost_root_index, host->machine_guid) != 0)
error("RRDHOST: %s() failed to delete machine guid '%s' from index", __FUNCTION__, host->machine_guid);
rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID);
}
}
// ----------------------------------------------------------------------------
// RRDHOST index by hostname
inline RRDHOST *rrdhost_find_by_hostname(const char *hostname) {
if(unlikely(!strcmp(hostname, "localhost")))
return localhost;
if(unlikely(!hash)) hash = simple_hash(hostname);
rrd_rdlock();
RRDHOST *host;
rrdhost_foreach_read(host) {
if(unlikely((hash == host->hash_hostname && !strcmp(hostname, host->hostname)))) {
rrd_unlock();
return host;
}
}
rrd_unlock();
return NULL;
return dictionary_get(rrdhost_root_index_hostname, hostname);
}
#define rrdhost_index_add(rrdhost) (RRDHOST *)avl_insert_lock(&(rrdhost_root_index), (avl_t *)(rrdhost))
#define rrdhost_index_del(rrdhost) (RRDHOST *)avl_remove_lock(&(rrdhost_root_index), (avl_t *)(rrdhost))
static inline RRDHOST *rrdhost_index_add_hostname(RRDHOST *host) {
if(!host->hostname) return host;
RRDHOST *ret_hostname = dictionary_set(rrdhost_root_index_hostname, rrdhost_hostname(host), host, sizeof(RRDHOST));
if(ret_hostname == host)
rrdhost_flag_set(host, RRDHOST_FLAG_INDEXED_HOSTNAME);
else {
rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_HOSTNAME);
error("RRDHOST: %s() host with hostname '%s' is already indexed", __FUNCTION__, rrdhost_hostname(host));
}
return host;
}
static inline void rrdhost_index_del_hostname(RRDHOST *host) {
if(unlikely(!host->hostname)) return;
if(rrdhost_flag_check(host, RRDHOST_FLAG_INDEXED_HOSTNAME)) {
if(dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host)) != 0)
error("RRDHOST: %s() failed to delete hostname '%s' from index", __FUNCTION__, rrdhost_hostname(host));
rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_HOSTNAME);
}
}
// ----------------------------------------------------------------------------
// RRDHOST - internal helpers
static inline void rrdhost_init_tags(RRDHOST *host, const char *tags) {
if(host->tags && tags && !strcmp(host->tags, tags))
if(host->tags && tags && !strcmp(rrdhost_tags(host), tags))
return;
void *old = (void *)host->tags;
host->tags = (tags && *tags)?strdupz(tags):NULL;
freez(old);
STRING *old = host->tags;
host->tags = string_strdupz((tags && *tags)?tags:NULL);
string_freez(old);
}
static inline void rrdhost_init_hostname(RRDHOST *host, const char *hostname) {
if(host->hostname && hostname && !strcmp(host->hostname, hostname))
if(unlikely(hostname && !*hostname)) hostname = NULL;
if(host->hostname && hostname && !strcmp(rrdhost_hostname(host), hostname))
return;
void *old = host->hostname;
host->hostname = strdupz(hostname?hostname:"localhost");
host->hash_hostname = simple_hash(host->hostname);
freez(old);
rrdhost_index_del_hostname(host);
STRING *old = host->hostname;
host->hostname = string_strdupz(hostname?hostname:"localhost");
string_freez(old);
rrdhost_index_add_hostname(host);
}
static inline void rrdhost_init_os(RRDHOST *host, const char *os) {
if(host->os && os && !strcmp(host->os, os))
if(host->os && os && !strcmp(rrdhost_os(host), os))
return;
void *old = (void *)host->os;
host->os = strdupz(os?os:"unknown");
freez(old);
STRING *old = host->os;
host->os = string_strdupz(os?os:"unknown");
string_freez(old);
}
static inline void rrdhost_init_timezone(RRDHOST *host, const char *timezone, const char *abbrev_timezone, int32_t utc_offset) {
if (host->timezone && timezone && !strcmp(host->timezone, timezone) && host->abbrev_timezone && abbrev_timezone &&
!strcmp(host->abbrev_timezone, abbrev_timezone) && host->utc_offset == utc_offset)
if (host->timezone && timezone && !strcmp(rrdhost_timezone(host), timezone) && host->abbrev_timezone && abbrev_timezone &&
!strcmp(rrdhost_abbrev_timezone(host), abbrev_timezone) && host->utc_offset == utc_offset)
return;
void *old = (void *)host->timezone;
host->timezone = strdupz((timezone && *timezone)?timezone:"unknown");
freez(old);
STRING *old = host->timezone;
host->timezone = string_strdupz((timezone && *timezone)?timezone:"unknown");
string_freez(old);
old = (void *)host->abbrev_timezone;
host->abbrev_timezone = strdupz((abbrev_timezone && *abbrev_timezone) ? abbrev_timezone : "UTC");
host->abbrev_timezone = string_strdupz((abbrev_timezone && *abbrev_timezone) ? abbrev_timezone : "UTC");
freez(old);
host->utc_offset = utc_offset;
}
static inline void rrdhost_init_machine_guid(RRDHOST *host, const char *machine_guid) {
strncpy(host->machine_guid, machine_guid, GUID_LEN);
host->machine_guid[GUID_LEN] = '\0';
host->hash_machine_guid = simple_hash(host->machine_guid);
}
void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *hostname,
const char *registry_hostname, const char *guid, const char *os, const char *tags,
void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode,
const char *registry_hostname, const char *os, const char *tags,
const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *program_name,
const char *program_version)
{
@ -151,18 +194,13 @@ void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory
host->rrd_update_every = update_every;
host->rrd_memory_mode = memory_mode;
rrdhost_init_hostname(host, hostname);
rrdhost_init_machine_guid(host, guid);
rrdhost_init_os(host, os);
rrdhost_init_timezone(host, tzone, abbrev_tzone, utc_offset);
rrdhost_init_tags(host, tags);
host->program_name = strdupz((program_name && *program_name) ? program_name : "unknown");
host->program_version = strdupz((program_version && *program_version) ? program_version : "unknown");
host->registry_hostname = strdupz((registry_hostname && *registry_hostname) ? registry_hostname : host->hostname);
host->program_name = string_strdupz((program_name && *program_name) ? program_name : "unknown");
host->program_version = string_strdupz((program_version && *program_version) ? program_version : "unknown");
host->registry_hostname = string_strdupz((registry_hostname && *registry_hostname) ? registry_hostname : rrdhost_hostname(host));
}
// ----------------------------------------------------------------------------
@ -202,9 +240,14 @@ RRDHOST *rrdhost_create(const char *hostname,
int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy);
RRDHOST *host = callocz(1, sizeof(RRDHOST));
set_host_properties(host, (update_every > 0)?update_every:1, memory_mode, hostname, registry_hostname, guid, os,
strncpy(host->machine_guid, guid, GUID_LEN);
host->machine_guid[GUID_LEN] = '\0';
set_host_properties(host, (update_every > 0)?update_every:1, memory_mode, registry_hostname, os,
tags, timezone, abbrev_timezone, utc_offset, program_name, program_version);
rrdhost_init_hostname(host, hostname);
host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);
host->health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled;
@ -237,10 +280,10 @@ RRDHOST *rrdhost_create(const char *hostname,
host->system_info = system_info;
avl_init_lock(&(host->rrdset_root_index), rrdset_compare);
avl_init_lock(&(host->rrdset_root_index_name), rrdset_compare_name);
avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare);
avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare);
host->rrdset_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
host->rrdset_root_index_name = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
host->rrdfamily_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
host->rrdvar_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
if(config_get_boolean(CONFIG_SECTION_DB, "delete obsolete charts files", 1))
rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS);
@ -250,8 +293,6 @@ RRDHOST *rrdhost_create(const char *hostname,
host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");
avl_init_lock(&(host->alarms_idx_health_log), alarm_compare_id);
avl_init_lock(&(host->alarms_idx_name), alarm_compare_name);
// ------------------------------------------------------------------------
// initialize health variables
@ -264,7 +305,7 @@ RRDHOST *rrdhost_create(const char *hostname,
long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max);
if(n < 10) {
error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", host->hostname, n, host->health_log.max);
error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", rrdhost_hostname(host), n, host->health_log.max);
config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max);
}
else
@ -293,7 +334,7 @@ RRDHOST *rrdhost_create(const char *hostname,
host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) {
int r = mkdir(host->cache_dir, 0775);
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, host->cache_dir);
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->cache_dir);
}
snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_varlib_dir, host->machine_guid);
@ -302,7 +343,7 @@ RRDHOST *rrdhost_create(const char *hostname,
if(host->health_enabled) {
int r = mkdir(host->varlib_dir, 0775);
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, host->varlib_dir);
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir);
}
}
@ -311,15 +352,15 @@ RRDHOST *rrdhost_create(const char *hostname,
snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir);
int r = mkdir(filename, 0775);
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, filename);
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename);
}
snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir);
host->health_log_filename = strdupz(filename);
snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir);
host->health_default_exec = strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
host->health_default_recipient = strdupz("root");
host->health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
host->health_default_recipient = string_strdupz("root");
// ------------------------------------------------------------------------
@ -331,10 +372,9 @@ RRDHOST *rrdhost_create(const char *hostname,
rrdhost_unlock(host);
}
RRDHOST *t = rrdhost_index_add(host);
RRDHOST *t = rrdhost_index_add_by_guid(host);
if(t != host) {
error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", host->hostname, host->machine_guid, t->hostname, t->machine_guid);
error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", rrdhost_hostname(host), host->machine_guid, rrdhost_hostname(t), t->machine_guid);
rrdhost_free(host, 1);
return NULL;
}
@ -376,7 +416,7 @@ RRDHOST *rrdhost_create(const char *hostname,
snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", host->cache_dir);
ret = mkdir(dbenginepath, 0775);
if (ret != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath);
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), dbenginepath);
else ret = 0; // succeed
if (is_legacy) {
// initialize legacy dbengine instance as needed
@ -403,7 +443,7 @@ RRDHOST *rrdhost_create(const char *hostname,
if (ret) { // check legacy or multihost initialization success
error(
"Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",
host->hostname, host->machine_guid, host->cache_dir);
rrdhost_hostname(host), host->machine_guid, host->cache_dir);
rrdhost_free(host, 1);
host = NULL;
//rrd_hosts_available++; //TODO: maybe we want this?
@ -426,17 +466,10 @@ RRDHOST *rrdhost_create(const char *hostname,
// ------------------------------------------------------------------------
// link it and add it to the index
if(is_localhost) {
host->next = localhost;
localhost = host;
}
else {
if(localhost) {
host->next = localhost->next;
localhost->next = host;
}
else localhost = host;
}
if(is_localhost)
DOUBLE_LINKED_LIST_PREPEND_UNSAFE(localhost, host, prev, next);
else
DOUBLE_LINKED_LIST_APPEND_UNSAFE(localhost, host, prev, next);
// ------------------------------------------------------------------------
// init new ML host and update system_info to let upstreams know
@ -466,14 +499,14 @@ RRDHOST *rrdhost_create(const char *hostname,
", health_log '%s'"
", alarms default handler '%s'"
", alarms default recipient '%s'"
, host->hostname
, host->registry_hostname
, rrdhost_hostname(host)
, rrdhost_registry_hostname(host)
, host->machine_guid
, host->os
, host->timezone
, (host->tags)?host->tags:""
, host->program_name
, host->program_version
, rrdhost_os(host)
, rrdhost_timezone(host)
, rrdhost_tags(host)
, rrdhost_program_name(host)
, rrdhost_program_version(host)
, host->rrd_update_every
, rrd_memory_mode_name(host->rrd_memory_mode)
, host->rrd_history_entries
@ -484,8 +517,8 @@ RRDHOST *rrdhost_create(const char *hostname,
, host->cache_dir
, host->varlib_dir
, host->health_log_filename
, host->health_default_exec
, host->health_default_recipient
, string2str(host->health_default_exec)
, string2str(host->health_default_recipient)
);
sql_store_host_system_info(&host->host_uuid, system_info);
@ -537,39 +570,36 @@ void rrdhost_update(RRDHOST *host
rrdhost_init_os(host, os);
rrdhost_init_timezone(host, timezone, abbrev_timezone, utc_offset);
freez(host->registry_hostname);
host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);
string_freez(host->registry_hostname);
host->registry_hostname = string_strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);
if(strcmp(host->hostname, hostname) != 0) {
info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname);
char *t = host->hostname;
host->hostname = strdupz(hostname);
host->hash_hostname = simple_hash(host->hostname);
freez(t);
if(strcmp(rrdhost_hostname(host), hostname) != 0) {
info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", rrdhost_hostname(host), hostname);
rrdhost_init_hostname(host, hostname);
}
if(strcmp(host->program_name, program_name) != 0) {
info("Host '%s' switched program name from '%s' to '%s'", host->hostname, host->program_name, program_name);
char *t = host->program_name;
host->program_name = strdupz(program_name);
freez(t);
if(strcmp(rrdhost_program_name(host), program_name) != 0) {
info("Host '%s' switched program name from '%s' to '%s'", rrdhost_hostname(host), rrdhost_program_name(host), program_name);
STRING *t = host->program_name;
host->program_name = string_strdupz(program_name);
string_freez(t);
}
if(strcmp(host->program_version, program_version) != 0) {
info("Host '%s' switched program version from '%s' to '%s'", host->hostname, host->program_version, program_version);
char *t = host->program_version;
host->program_version = strdupz(program_version);
freez(t);
if(strcmp(rrdhost_program_version(host), program_version) != 0) {
info("Host '%s' switched program version from '%s' to '%s'", rrdhost_hostname(host), rrdhost_program_version(host), program_version);
STRING *t = host->program_version;
host->program_version = string_strdupz(program_version);
string_freez(t);
}
if(host->rrd_update_every != update_every)
error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every);
error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", rrdhost_hostname(host), host->rrd_update_every, update_every);
if(host->rrd_history_entries < history)
error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history);
error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", rrdhost_hostname(host), host->rrd_history_entries, history);
if(host->rrd_memory_mode != mode)
error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
// update host tags
rrdhost_init_tags(host, tags);
@ -591,12 +621,12 @@ void rrdhost_update(RRDHOST *host
if (host != localhost) {
r = mkdir(host->varlib_dir, 0775);
if (r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, host->varlib_dir);
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir);
}
snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir);
r = mkdir(filename, 0775);
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, filename);
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename);
rrdhost_wrlock(host);
health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
@ -621,7 +651,7 @@ void rrdhost_update(RRDHOST *host
rrd_hosts_available++;
ml_new_host(host);
rrdhost_load_rrdcontext_data(host);
info("Host %s is not in archived mode anymore", host->hostname);
info("Host %s is not in archived mode anymore", rrdhost_hostname(host));
}
return;
@ -652,11 +682,11 @@ RRDHOST *rrdhost_find_or_create(
debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid);
rrd_wrlock();
RRDHOST *host = rrdhost_find_by_guid(guid, 0);
RRDHOST *host = rrdhost_find_by_guid(guid);
if (unlikely(host && RRD_MEMORY_MODE_DBENGINE != mode && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
/* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */
error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.",
host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
rrdhost_free(host, 1);
host = NULL;
}
@ -739,7 +769,7 @@ void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host) {
restart_after_removal:
rrdhost_foreach_write(host) {
if(rrdhost_should_be_removed(host, protected_host, now)) {
info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid);
info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", rrdhost_hostname(host), host->machine_guid);
if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST)
#ifdef ENABLE_DBENGINE
@ -761,6 +791,7 @@ restart_after_removal:
// RRDHOST global / startup initialization
int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
rrdhost_init();
if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
@ -940,19 +971,19 @@ unittest:
// there are only used when NETDATA_INTERNAL_CHECKS is set
void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line) {
debug(D_RRDHOST, "Checking read lock on host '%s'", host->hostname);
debug(D_RRDHOST, "Checking read lock on host '%s'", rrdhost_hostname(host));
int ret = netdata_rwlock_trywrlock(&host->rrdhost_rwlock);
if(ret == 0)
fatal("RRDHOST '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", host->hostname, function, line, file);
fatal("RRDHOST '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", rrdhost_hostname(host), function, line, file);
}
void __rrdhost_check_wrlock(RRDHOST *host, const char *file, const char *function, const unsigned long line) {
debug(D_RRDHOST, "Checking write lock on host '%s'", host->hostname);
debug(D_RRDHOST, "Checking write lock on host '%s'", rrdhost_hostname(host));
int ret = netdata_rwlock_tryrdlock(&host->rrdhost_rwlock);
if(ret == 0)
fatal("RRDHOST '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", host->hostname, function, line, file);
fatal("RRDHOST '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", rrdhost_hostname(host), function, line, file);
}
void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line) {
@ -1051,7 +1082,7 @@ void rrdhost_free(RRDHOST *host, bool force) {
if(!host) return;
if (netdata_exit || force)
info("Freeing all memory for host '%s'...", host->hostname);
info("Freeing all memory for host '%s'...", rrdhost_hostname(host));
rrd_check_wrlock(); // make sure the RRDs are write locked
@ -1084,8 +1115,8 @@ void rrdhost_free(RRDHOST *host, bool force) {
freez(host->exporting_flags);
while(host->alarms)
rrdcalc_unlink_and_free(host, host->alarms);
while(host->host_alarms)
rrdcalc_unlink_and_free(host, host->host_alarms);
RRDCALC *rc,*nc;
for(rc = host->alarms_with_foreach; rc ; rc = nc) {
@ -1094,18 +1125,11 @@ void rrdhost_free(RRDHOST *host, bool force) {
}
host->alarms_with_foreach = NULL;
while(host->templates)
rrdcalctemplate_unlink_and_free(host, host->templates);
while(host->alarms_templates)
rrdcalctemplate_unlink_and_free(host, host->alarms_templates);
RRDCALCTEMPLATE *rt,*next;
for(rt = host->alarms_template_with_foreach; rt ; rt = next) {
next = rt->next;
rrdcalctemplate_free(rt);
}
host->alarms_template_with_foreach = NULL;
debug(D_RRD_CALLS, "RRDHOST: Cleaning up remaining host variables for host '%s'", host->hostname);
rrdvar_free_remaining_variables(host, &host->rrdvar_root_index);
debug(D_RRD_CALLS, "RRDHOST: Cleaning up remaining host variables for host '%s'", rrdhost_hostname(host));
rrdvar_free_remaining_variables(host, host->rrdvar_root_index);
health_alarm_log_free(host);
@ -1119,7 +1143,7 @@ void rrdhost_free(RRDHOST *host, bool force) {
#endif
if (!netdata_exit && !force) {
info("Setting archive mode for host '%s'...", host->hostname);
info("Setting archive mode for host '%s'...", rrdhost_hostname(host));
rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
rrdhost_unlock(host);
return;
@ -1143,24 +1167,13 @@ void rrdhost_free(RRDHOST *host, bool force) {
// ------------------------------------------------------------------------
// remove it from the indexes
if(rrdhost_index_del(host) != host)
error("RRDHOST '%s' removed from index, deleted the wrong entry.", host->hostname);
rrdhost_index_del_hostname(host);
rrdhost_index_del_by_guid(host);
// ------------------------------------------------------------------------
// unlink it from the host
if(host == localhost) {
localhost = host->next;
}
else {
// find the previous one
RRDHOST *h;
for(h = localhost; h && h->next != host ; h = h->next) ;
// bypass it
if(h) h->next = host->next;
else error("Request to free RRDHOST '%s': cannot find it", host->hostname);
}
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(localhost, host, prev, next);
// ------------------------------------------------------------------------
// free it
@ -1168,13 +1181,13 @@ void rrdhost_free(RRDHOST *host, bool force) {
pthread_mutex_destroy(&host->aclk_state_lock);
freez(host->aclk_state.claimed_id);
freez(host->aclk_state.prev_claimed_id);
freez((void *)host->tags);
string_freez(host->tags);
rrdlabels_destroy(host->host_labels);
freez((void *)host->os);
freez((void *)host->timezone);
freez((void *)host->abbrev_timezone);
freez(host->program_version);
freez(host->program_name);
string_freez(host->os);
string_freez(host->timezone);
string_freez(host->abbrev_timezone);
string_freez(host->program_name);
string_freez(host->program_version);
rrdhost_system_info_free(host->system_info);
freez(host->cache_dir);
freez(host->varlib_dir);
@ -1186,19 +1199,24 @@ void rrdhost_free(RRDHOST *host, bool force) {
freez(host->destinations);
host->destinations = tmp_destination;
}
freez(host->health_default_exec);
freez(host->health_default_recipient);
string_freez(host->health_default_exec);
string_freez(host->health_default_recipient);
freez(host->health_log_filename);
freez(host->hostname);
freez(host->registry_hostname);
string_freez(host->registry_hostname);
simple_pattern_free(host->rrdpush_send_charts_matching);
rrdhost_unlock(host);
netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock);
netdata_rwlock_destroy(&host->rrdhost_rwlock);
freez(host->node_id);
dictionary_destroy(host->rrdset_root_index);
dictionary_destroy(host->rrdset_root_index_name);
dictionary_destroy(host->rrdfamily_root_index);
dictionary_destroy(host->rrdvar_root_index);
rrdhost_destroy_rrdcontexts(host);
string_freez(host->hostname);
freez(host);
#ifdef ENABLE_ACLK
if (wc)
@ -1209,9 +1227,14 @@ void rrdhost_free(RRDHOST *host, bool force) {
void rrdhost_free_all(void) {
rrd_wrlock();
/* Make sure child-hosts are released before the localhost. */
while(localhost->next) rrdhost_free(localhost->next, 1);
rrdhost_free(localhost, 1);
while(localhost && localhost->next)
rrdhost_free(localhost->next, 1);
if(localhost)
rrdhost_free(localhost, 1);
rrd_unlock();
}
@ -1221,7 +1244,7 @@ void rrdhost_free_all(void) {
void rrdhost_save_charts(RRDHOST *host) {
if(!host) return;
info("Saving/Closing database of host '%s'...", host->hostname);
info("Saving/Closing database of host '%s'...", rrdhost_hostname(host));
RRDSET *st;
@ -1302,7 +1325,7 @@ static void rrdhost_load_auto_labels(void) {
add_aclk_host_labels();
rrdlabels_add(
labels, "_is_parent", (localhost->next || configured_as_parent()) ? "true" : "false", RRDLABEL_SRC_AUTO);
labels, "_is_parent", (rrdhost_hosts_available() > 1 || configured_as_parent()) ? "true" : "false", RRDLABEL_SRC_AUTO);
if (localhost->rrdpush_send_destination)
rrdlabels_add(labels, "_streams_to", localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO);
@ -1383,7 +1406,7 @@ void reload_host_labels(void) {
void rrdhost_delete_charts(RRDHOST *host) {
if(!host) return;
info("Deleting database of host '%s'...", host->hostname);
info("Deleting database of host '%s'...", rrdhost_hostname(host));
RRDSET *st;
@ -1408,7 +1431,7 @@ void rrdhost_delete_charts(RRDHOST *host) {
void rrdhost_cleanup_charts(RRDHOST *host) {
if(!host) return;
info("Cleaning up database of host '%s'...", host->hostname);
info("Cleaning up database of host '%s'...", rrdhost_hostname(host));
RRDSET *st;
uint32_t rrdhost_delete_obsolete_charts = rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS);
@ -1497,8 +1520,8 @@ restart_after_removal:
RRDDIM *rd, *last;
rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED);
while (st->variables) rrdsetvar_free(st->variables);
while (st->alarms) rrdsetcalc_unlink(st->alarms);
while (st->variables) rrdsetvar_free(st->variables);
while (st->alarms) rrdsetcalc_unlink(st->alarms);
rrdset_wrlock(st);
for (rd = st->dimensions, last = NULL ; likely(rd) ; ) {
if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
@ -1555,8 +1578,8 @@ restart_after_removal:
}
rrdset_unlock(st);
debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", host->hostname, st->id);
rrdvar_free_remaining_variables(host, &st->rrdvar_root_index);
debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", rrdhost_hostname(host), rrdset_id(st));
rrdvar_free_remaining_variables(host, st->rrdvar_root_index);
rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
@ -1761,46 +1784,6 @@ int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, ch
return res;
}
/**
* Alarm Compare ID
*
* Callback function used with the binary trees to compare the id of RRDCALC
*
* @param a a pointer to the RRDCAL item to insert,compare or update the binary tree
* @param b the pointer to the binary tree.
*
* @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b.
*/
int alarm_compare_id(void *a, void *b) {
register uint32_t hash1 = ((RRDCALC *)a)->id;
register uint32_t hash2 = ((RRDCALC *)b)->id;
if(hash1 < hash2) return -1;
else if(hash1 > hash2) return 1;
return 0;
}
/**
* Alarm Compare NAME
*
* Callback function used with the binary trees to compare the name of RRDCALC
*
* @param a a pointer to the RRDCAL item to insert,compare or update the binary tree
* @param b the pointer to the binary tree.
*
* @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b.
*/
int alarm_compare_name(void *a, void *b) {
RRDCALC *in1 = (RRDCALC *)a;
RRDCALC *in2 = (RRDCALC *)b;
if(in1->hash < in2->hash) return -1;
else if(in1->hash > in2->hash) return 1;
return strcmp(in1->name,in2->name);
}
// Added for gap-filling, if this proves to be a bottleneck in large-scale systems then we will need to cache
// the last entry times as the metric updates, but let's see if it is a problem first.
time_t rrdhost_last_entry_t(RRDHOST *h) {

24
database/rrdlabels.c
View File

@ -503,7 +503,7 @@ static void rrdlabel_conflict_callback(const char *name, void *oldvalue, void *n
RRDLABEL *lbold = (RRDLABEL *)oldvalue;
RRDLABEL *lbnew = (RRDLABEL *)newvalue;
if(lbold->label_value == lbnew->label_value || strcmp(string2str(lbold->label_value), string2str(lbnew->label_value)) == 0) {
if(lbold->label_value == lbnew->label_value) {
// they are the same
lbold->label_source |= lbnew->label_source;
lbold->label_source |= RRDLABEL_FLAG_OLD;
@ -937,9 +937,20 @@ int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each,
return dictionary_walkthrough_read(labels, label_to_buffer_callback, (void *)&tmp);
}
struct label_str {
BUFFER *sql;
int count;
char uuid_str[UUID_STR_LEN];
};
static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
RRDSET *st = (RRDSET *)data;
sql_store_chart_label(st->chart_uuid, (int)ls, (char *)name, (char *)value);
struct label_str *lb = data;
if (unlikely(!lb->count))
buffer_sprintf(lb->sql, "INSERT OR REPLACE INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES ");
else
buffer_strcat(lb->sql, ", ");
buffer_sprintf(lb->sql, "(u2h('%s'), %d,'%s','%s', unixepoch())", lb->uuid_str, ls, name, value);
lb->count++;
return 1;
}
@ -953,7 +964,12 @@ void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
rrdcalc_update_rrdlabels(st);
// TODO - we should also cleanup sqlite from old new_rrdlabels that have been removed
rrdlabels_walkthrough_read(st->state->chart_labels, chart_label_store_to_sql_callback, st);
BUFFER *sql_buf = buffer_create(1024);
struct label_str tmp = {.sql = sql_buf, .count = 0 };
uuid_unparse_lower(*st->chart_uuid, tmp.uuid_str);
rrdlabels_walkthrough_read(st->state->chart_labels, chart_label_store_to_sql_callback, &tmp);
db_execute(buffer_tostring(sql_buf));
buffer_free(sql_buf);
}
// ----------------------------------------------------------------------------

451
database/rrdset.c
View File

@ -5,95 +5,85 @@
#include <sched.h>
void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line) {
debug(D_RRD_CALLS, "Checking read lock on chart '%s'", st->id);
debug(D_RRD_CALLS, "Checking read lock on chart '%s'", rrdset_id(st));
int ret = netdata_rwlock_trywrlock(&st->rrdset_rwlock);
if(ret == 0)
fatal("RRDSET '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", st->id, function, line, file);
fatal("RRDSET '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", rrdset_id(st), function, line, file);
}
void __rrdset_check_wrlock(RRDSET *st, const char *file, const char *function, const unsigned long line) {
debug(D_RRD_CALLS, "Checking write lock on chart '%s'", st->id);
debug(D_RRD_CALLS, "Checking write lock on chart '%s'", rrdset_id(st));
int ret = netdata_rwlock_tryrdlock(&st->rrdset_rwlock);
if(ret == 0)
fatal("RRDSET '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", st->id, function, line, file);
fatal("RRDSET '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", rrdset_id(st), function, line, file);
}
// ----------------------------------------------------------------------------
// RRDSET index
int rrdset_compare(void* a, void* b) {
if(((RRDSET *)a)->hash < ((RRDSET *)b)->hash) return -1;
else if(((RRDSET *)a)->hash > ((RRDSET *)b)->hash) return 1;
else return strcmp(((RRDSET *)a)->id, ((RRDSET *)b)->id);
static inline void rrdset_index_add(RRDHOST *host, RRDSET *st) {
if(likely(dictionary_set(host->rrdset_root_index, rrdset_id(st), st, sizeof(RRDSET)) == st)) {
rrdset_flag_set(st, RRDSET_FLAG_INDEXED_ID);
}
else {
rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID);
error("RRDSET: %s() attempted to index duplicate object with key '%s'", __FUNCTION__, rrdset_id(st));
}
}
static RRDSET *rrdset_index_find(RRDHOST *host, const char *id, uint32_t hash) {
RRDSET tmp;
strncpyz(tmp.id, id, RRD_ID_LENGTH_MAX);
tmp.hash = (hash)?hash:simple_hash(tmp.id);
static inline void rrdset_index_del(RRDHOST *host, RRDSET *st) {
if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID)) {
if(likely(dictionary_del(host->rrdset_root_index, rrdset_id(st)) == 0))
rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID);
else
error("RRDSET: %s() attempted to delete non-indexed object with key '%s'", __FUNCTION__, rrdset_id(st));
}
}
return (RRDSET *)avl_search_lock(&(host->rrdset_root_index), (avl_t *) &tmp);
static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) {
return dictionary_get(host->rrdset_root_index, id);
}
// ----------------------------------------------------------------------------
// RRDSET name index
#define rrdset_from_avlname(avlname_ptr) ((RRDSET *)((avlname_ptr) - offsetof(RRDSET, avlname)))
int rrdset_compare_name(void* a, void* b) {
RRDSET *A = rrdset_from_avlname(a);
RRDSET *B = rrdset_from_avlname(b);
// fprintf(stderr, "COMPARING: %s with %s\n", A->name, B->name);
if(A->hash_name < B->hash_name) return -1;
else if(A->hash_name > B->hash_name) return 1;
else return strcmp(A->name, B->name);
static inline void rrdset_index_add_name(RRDHOST *host, RRDSET *st) {
if(likely(dictionary_set(host->rrdset_root_index_name, rrdset_name(st), st, sizeof(RRDSET)) == st)) {
rrdset_flag_set(st, RRDSET_FLAG_INDEXED_NAME);
}
else {
rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME);
error("RRDSET: %s() attempted to index duplicate object with key '%s'", __FUNCTION__, rrdset_name(st));
}
}
RRDSET *rrdset_index_add_name(RRDHOST *host, RRDSET *st) {
void *result;
// fprintf(stderr, "ADDING: %s (name: %s)\n", st->id, st->name);
result = avl_insert_lock(&host->rrdset_root_index_name, (avl_t *) (&st->avlname));
if(result) return rrdset_from_avlname(result);
return NULL;
static inline void rrdset_index_del_name(RRDHOST *host, RRDSET *st) {
if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID)) {
if(likely(dictionary_del(host->rrdset_root_index_name, rrdset_name(st)) != 0))
rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME);
else
error("RRDSET: %s() attempted to delete non-index object with key '%s'", __FUNCTION__, rrdset_name(st));
}
}
RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st) {
void *result;
// fprintf(stderr, "DELETING: %s (name: %s)\n", st->id, st->name);
result = (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index_name), (avl_t *)(&st->avlname));
if(result) return rrdset_from_avlname(result);
return NULL;
static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) {
return dictionary_get(host->rrdset_root_index_name, name);
}
// ----------------------------------------------------------------------------
// RRDSET - find charts
static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name, uint32_t hash) {
void *result = NULL;
RRDSET tmp;
tmp.name = name;
tmp.hash_name = (hash)?hash:simple_hash(tmp.name);
result = avl_search_lock(&host->rrdset_root_index_name, (avl_t *) (&(tmp.avlname)));
if(result) return rrdset_from_avlname(result);
return NULL;
}
inline RRDSET *rrdset_find(RRDHOST *host, const char *id) {
debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, host->hostname);
RRDSET *st = rrdset_index_find(host, id, 0);
debug(D_RRD_CALLS, "rrdset_find() for chart '%s' in host '%s'", id, rrdhost_hostname(host));
RRDSET *st = rrdset_index_find(host, id);
return(st);
}
inline RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id) {
debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, host->hostname);
debug(D_RRD_CALLS, "rrdset_find_bytype() for chart '%s.%s' in host '%s'", type, id, rrdhost_hostname(host));
char buf[RRD_ID_LENGTH_MAX + 1];
strncpyz(buf, type, RRD_ID_LENGTH_MAX - 1);
@ -105,8 +95,8 @@ inline RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *i
}
inline RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) {
debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, host->hostname);
RRDSET *st = rrdset_index_find_name(host, name, 0);
debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, rrdhost_hostname(host));
RRDSET *st = rrdset_index_find_name(host, name);
return(st);
}
@ -129,32 +119,32 @@ char *rrdset_strncpyz_name(char *to, const char *from, size_t length) {
}
int rrdset_set_name(RRDSET *st, const char *name) {
if(unlikely(st->name && !strcmp(st->name, name)))
if(unlikely(!strcmp(rrdset_name(st), name)))
return 1;
RRDHOST *host = st->rrdhost;
debug(D_RRD_CALLS, "rrdset_set_name() old: '%s', new: '%s'", st->name?st->name:"", name);
debug(D_RRD_CALLS, "rrdset_set_name() old: '%s', new: '%s'", rrdset_name(st), name);
char full_name[RRD_ID_LENGTH_MAX + 1];
char sanitized_name[CONFIG_MAX_VALUE + 1];
char new_name[CONFIG_MAX_VALUE + 1];
snprintfz(full_name, RRD_ID_LENGTH_MAX, "%s.%s", st->type, name);
snprintfz(full_name, RRD_ID_LENGTH_MAX, "%s.%s", rrdset_type(st), name);
rrdset_strncpyz_name(sanitized_name, full_name, CONFIG_MAX_VALUE);
strncpyz(new_name, sanitized_name, CONFIG_MAX_VALUE);
if(rrdset_index_find_name(host, new_name, 0)) {
debug(D_RRD_CALLS, "RRDSET: chart name '%s' on host '%s' already exists.", new_name, host->hostname);
if(!strcmp(st->id, full_name) && !st->name) {
if(rrdset_index_find_name(host, new_name)) {
debug(D_RRD_CALLS, "RRDSET: chart name '%s' on host '%s' already exists.", new_name, rrdhost_hostname(host));
if(!strcmp(rrdset_id(st), full_name) && !st->name) {
unsigned i = 1;
do {
snprintfz(new_name, CONFIG_MAX_VALUE, "%s_%u", sanitized_name, i);
i++;
} while (rrdset_index_find_name(host, new_name, 0));
} while (rrdset_index_find_name(host, new_name));
info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, host->hostname);
info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, rrdhost_hostname(host));
} else {
return 0;
}
@ -162,13 +152,12 @@ int rrdset_set_name(RRDSET *st, const char *name) {
if(st->name) {
rrdset_index_del_name(host, st);
st->name = strdupz(new_name);
st->hash_name = simple_hash(st->name);
string_freez(st->name);
st->name = string_strdupz(new_name);
rrdsetvar_rename_all(st);
}
else {
st->name = strdupz(new_name);
st->hash_name = simple_hash(st->name);
st->name = string_strdupz(new_name);
}
rrdset_wrlock(st);
@ -177,8 +166,7 @@ int rrdset_set_name(RRDSET *st, const char *name) {
rrddimvar_rename_all(rd);
rrdset_unlock(st);
if(unlikely(rrdset_index_add_name(host, st) != st))
error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name);
rrdset_index_add_name(host, st);
rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND);
rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE);
@ -192,12 +180,13 @@ int rrdset_set_name(RRDSET *st, const char *name) {
inline void rrdset_is_obsolete(RRDSET *st) {
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))) {
info("Cannot obsolete already archived chart %s", st->name);
info("Cannot obsolete already archived chart %s", rrdset_name(st));
return;
}
if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE);
st->last_accessed_time = now_realtime_sec();
st->rrdhost->obsolete_charts_count++;
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
@ -212,6 +201,7 @@ inline void rrdset_is_obsolete(RRDSET *st) {
inline void rrdset_isnot_obsolete(RRDSET *st) {
if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
st->last_accessed_time = now_realtime_sec();
st->rrdhost->obsolete_charts_count--;
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
@ -239,12 +229,12 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) {
if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {
#ifdef NETDATA_INTERNAL_CHECKS
info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").",
rd->name,
st->name,
host->hostname,
rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm),
rd->multiplier, multiplier,
rd->divisor, divisor
rrddim_name(rd),
rrdset_name(st),
rrdhost_hostname(host),
rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm),
rd->multiplier, multiplier,
rd->divisor, divisor
);
#endif
rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS);
@ -261,7 +251,7 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) {
// RRDSET - reset a chart
void rrdset_reset(RRDSET *st) {
debug(D_RRD_CALLS, "rrdset_reset() %s", st->name);
debug(D_RRD_CALLS, "rrdset_reset() %s", rrdset_name(st));
st->last_collected_time.tv_sec = 0;
st->last_collected_time.tv_usec = 0;
@ -346,9 +336,7 @@ void rrdset_free(RRDSET *st) {
// ------------------------------------------------------------------------
// remove it from the indexes
if(unlikely(rrdset_index_del(host, st) != st))
error("RRDSET: INTERNAL ERROR: attempt to remove from index chart '%s', removed a different chart.", st->id);
rrdset_index_del(host, st);
rrdset_index_del_name(host, st);
// ------------------------------------------------------------------------
@ -361,29 +349,18 @@ void rrdset_free(RRDSET *st) {
/* We must free all connected alarms here in case this has been an ephemeral chart whose alarm was
* created by a template. This leads to an effective memory leak, which cannot be detected since the
* alarms will still be connected to the host, and freed during shutdown. */
while(st->alarms) rrdcalc_unlink_and_free(st->rrdhost, st->alarms);
while(st->dimensions) rrddim_free(st, st->dimensions);
while(st->alarms) rrdcalc_unlink_and_free(st->rrdhost, st->alarms);
while(st->dimensions) rrddim_free(st, st->dimensions);
rrdfamily_free(host, st->rrdfamily);
debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", host->hostname, st->id);
rrdvar_free_remaining_variables(host, &st->rrdvar_root_index);
debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", rrdhost_hostname(host), rrdset_id(st));
rrdvar_free_remaining_variables(host, st->rrdvar_root_index);
// ------------------------------------------------------------------------
// unlink it from the host
if(st == host->rrdset_root) {
host->rrdset_root = st->next;
}
else {
// find the previous one
RRDSET *s;
for(s = host->rrdset_root; s && s->next != st ; s = s->next) ;
// bypass it
if(s) s->next = st->next;
else error("Request to free RRDSET '%s': cannot find it under host '%s'", st->id, host->hostname);
}
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->rrdset_root, st, prev, next);
rrdset_unlock(st);
@ -395,24 +372,28 @@ void rrdset_free(RRDSET *st) {
netdata_rwlock_destroy(&st->rrdset_rwlock);
// free directly allocated members
freez((void *)st->name);
freez(st->type);
freez(st->family);
freez(st->title);
freez(st->units);
freez(st->context);
freez(st->cache_dir);
freez(st->plugin_name);
freez(st->module_name);
freez(st->state->old_title);
freez(st->state->old_units);
freez(st->state->old_context);
rrdset_memory_file_free(st);
rrdlabels_destroy(st->state->chart_labels);
// free directly allocated members
dictionary_destroy(st->rrddim_root_index);
dictionary_destroy(st->rrdvar_root_index);
string_freez(st->id);
string_freez(st->name);
string_freez(st->type);
string_freez(st->family);
string_freez(st->title);
string_freez(st->units);
string_freez(st->context);
string_freez(st->plugin_name);
string_freez(st->module_name);
freez(st->cache_dir);
freez(st->state);
freez(st->chart_uuid);
rrdset_memory_file_free(st);
freez(st);
}
@ -430,7 +411,7 @@ void rrdset_delete_files(RRDSET *st) {
RRDDIM *rd;
rrdset_check_rdlock(st);
info("Deleting chart '%s' ('%s') from disk...", st->id, st->name);
info("Deleting chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
const char *cache_filename = rrdset_cache_filename(st);
@ -440,7 +421,7 @@ void rrdset_delete_files(RRDSET *st) {
error("Cannot delete chart header file '%s'", cache_filename);
}
else
error("Cannot find the cache filename of chart '%s'", st->id);
error("Cannot find the cache filename of chart '%s'", rrdset_id(st));
}
rrddim_foreach_read(rd, st) {
@ -460,7 +441,7 @@ void rrdset_delete_obsolete_dimensions(RRDSET *st) {
rrdset_check_rdlock(st);
info("Deleting dimensions of chart '%s' ('%s') from disk...", st->id, st->name);
info("Deleting dimensions of chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
rrddim_foreach_read(rd, st) {
if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
@ -490,9 +471,9 @@ static inline RRDSET *rrdset_find_on_create(RRDHOST *host, const char *fullid) {
static inline void rrdset_update_permanent_labels(RRDSET *st) {
if(!st->state || !st->state->chart_labels) return;
rrdlabels_add(st->state->chart_labels, "_collect_plugin", st->plugin_name, RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT);
rrdlabels_add(st->state->chart_labels, "_collect_module", st->module_name, RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT);
rrdlabels_add(st->state->chart_labels, "_instance_family", st->family, RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT);
rrdlabels_add(st->state->chart_labels, "_collect_plugin", rrdset_plugin_name(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT);
rrdlabels_add(st->state->chart_labels, "_collect_module", rrdset_module_name(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT);
rrdlabels_add(st->state->chart_labels, "_instance_family",rrdset_family(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT);
}
RRDSET *rrdset_create_custom(
@ -549,6 +530,7 @@ RRDSET *rrdset_create_custom(
char fullid[RRD_ID_LENGTH_MAX + 1];
snprintfz(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
json_fix_string(fullid);
int changed_from_archived_to_active = 0;
RRDSET *st = rrdset_find_on_create(host, fullid);
@ -559,11 +541,9 @@ RRDSET *rrdset_create_custom(
changed_from_archived_to_active = 1;
mark_rebuild |= META_CHART_ACTIVATED;
}
char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_context = NULL,
*old_title_v = NULL, *old_context_v = NULL, *old_units_v = NULL, *old_units = NULL;
int rc;
if(unlikely(name))
int rc;
if(unlikely(name && *name))
rc = rrdset_set_name(st, name);
else
rc = rrdset_set_name(st, id);
@ -580,91 +560,61 @@ RRDSET *rrdset_create_custom(
mark_rebuild |= META_CHART_UPDATED;
}
if (plugin && st->plugin_name) {
if (unlikely(strcmp(plugin, st->plugin_name))) {
old_plugin = st->plugin_name;
st->plugin_name = strdupz(plugin);
if(plugin && *plugin) {
STRING *old_plugin = st->plugin_name;
st->plugin_name = rrd_string_strdupz(plugin);
if (old_plugin != st->plugin_name)
mark_rebuild |= META_PLUGIN_UPDATED;
}
} else {
if (plugin != st->plugin_name) { // one is NULL?
old_plugin = st->plugin_name;
st->plugin_name = plugin ? strdupz(plugin) : NULL;
mark_rebuild |= META_PLUGIN_UPDATED;
}
string_freez(old_plugin);
}
if (module && st->module_name) {
if (unlikely(strcmp(module, st->module_name))) {
old_module = st->module_name;
st->module_name = strdupz(module);
if(module && *module) {
STRING *old_module = st->module_name;
st->module_name = rrd_string_strdupz(module);
if (old_module != st->module_name)
mark_rebuild |= META_MODULE_UPDATED;
}
} else {
if (module != st->module_name) {
if (st->module_name && *st->module_name) {
old_module = st->module_name;
st->module_name = module ? strdupz(module) : NULL;
mark_rebuild |= META_MODULE_UPDATED;
}
}
string_freez(old_module);
}
if (unlikely(title && st->state->old_title && strcmp(st->state->old_title, title))) {
char *new_title = strdupz(title);
old_title_v = st->state->old_title;
st->state->old_title = strdupz(title);
json_fix_string(new_title);
old_title = st->title;
st->title = new_title;
mark_rebuild |= META_CHART_UPDATED;
if(title && *title) {
STRING *old_title = st->title;
st->title = rrd_string_strdupz(title);
if(old_title != st->title)
mark_rebuild |= META_CHART_UPDATED;
string_freez(old_title);
}
if (unlikely(units && st->state->old_units && strcmp(st->state->old_units, units))) {
char *new_units = strdupz(units);
old_units_v = st->state->old_units;
st->state->old_units = strdupz(units);
json_fix_string(new_units);
old_units= st->units;
st->units = new_units;
mark_rebuild |= META_CHART_UPDATED;
if(units && *units) {
STRING *old_units = st->units;
st->units = rrd_string_strdupz(units);
if(old_units != st->units)
mark_rebuild |= META_CHART_UPDATED;
string_freez(old_units);
}
if(context && *context) {
STRING *old_context = st->context;
st->context = rrd_string_strdupz(context);
if(old_context != st->context)
mark_rebuild |= META_CHART_UPDATED;
string_freez(old_context);
}
if (st->chart_type != chart_type) {
st->chart_type = chart_type;
mark_rebuild |= META_CHART_UPDATED;
}
if (unlikely(context && st->state->old_context && strcmp(st->state->old_context, context))) {
char *new_context = strdupz(context);
old_context_v = st->state->old_context;
st->state->old_context = strdupz(context);
json_fix_string(new_context);
old_context = st->context;
st->context = new_context;
st->hash_context = simple_hash(st->context);
mark_rebuild |= META_CHART_UPDATED;
}
if (mark_rebuild) {
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
freez(old_plugin);
freez(old_module);
freez(old_title);
freez(old_units);
freez(old_context);
freez(old_title_v);
freez(old_units_v);
freez(old_context_v);
if (mark_rebuild != META_CHART_ACTIVATED) {
info("Collector updated metadata for chart %s", st->id);
info("Collector updated metadata for chart %s", rrdset_id(st));
sched_yield();
}
}
if (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED)) {
debug(D_METADATALOG, "CHART [%s] metadata updated", st->id);
int rc = update_chart_metadata(st->chart_uuid, st, id, name);
debug(D_METADATALOG, "CHART [%s] metadata updated", rrdset_id(st));
rc = update_chart_metadata(st->chart_uuid, st, id, name);
if (unlikely(rc))
error_report("Failed to update chart metadata in the database");
@ -719,8 +669,7 @@ RRDSET *rrdset_create_custom(
st = callocz(1, sizeof(RRDSET));
st->state = callocz(1, sizeof(*st->state));
strcpy(st->id, fullid);
st->hash = simple_hash(st->id);
st->id = string_strdupz(fullid); // fullid is already json_fix'ed
st->rrdhost = host;
st->cache_dir = cache_dir;
@ -729,29 +678,23 @@ RRDSET *rrdset_create_custom(
if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP) {
if(!rrdset_memory_load_or_create_map_save(st, memory_mode)) {
info("Failed to use memory mode %s for chart '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name);
info("Failed to use memory mode %s for chart '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st));
memory_mode = RRD_MEMORY_MODE_RAM;
}
}
st->rrd_memory_mode = memory_mode;
st->plugin_name = plugin?strdupz(plugin):NULL;
st->module_name = module?strdupz(module):NULL;
st->plugin_name = rrd_string_strdupz(plugin);
st->module_name = rrd_string_strdupz(module);
st->chart_type = chart_type;
st->type = strdupz(type);
st->family = family ? strdupz(family) : strdupz(st->type);
json_fix_string(st->family);
st->type = rrd_string_strdupz(type);
st->family = family ? rrd_string_strdupz(family) : string_dup(st->type);
st->state->is_ar_chart = strcmp(st->id, ML_ANOMALY_RATES_CHART_ID) == 0;
st->state->is_ar_chart = strcmp(rrdset_id(st), ML_ANOMALY_RATES_CHART_ID) == 0;
st->units = units ? strdupz(units) : strdupz("");
st->state->old_units = strdupz(st->units);
json_fix_string(st->units);
st->units = rrd_string_strdupz(units);
st->context = context ? strdupz(context) : strdupz(st->id);
st->state->old_context = strdupz(st->context);
json_fix_string(st->context);
st->hash_context = simple_hash(st->context);
st->context = (context && *context) ? rrd_string_strdupz(context) : string_dup(st->id);
st->priority = priority;
@ -762,8 +705,17 @@ RRDSET *rrdset_create_custom(
st->gap_when_lost_iterations_above = (int) (gap_when_lost_iterations_above + 2);
avl_init_lock(&st->dimensions_index, rrddim_compare);
avl_init_lock(&st->rrdvar_root_index, rrdvar_compare);
st->rrddim_root_index = dictionary_create(
DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
| DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE
| DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
);
st->rrdvar_root_index = dictionary_create(
DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
| DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE
| DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
);
netdata_rwlock_init(&st->rrdset_rwlock);
st->state->chart_labels = rrdlabels_create();
@ -776,14 +728,11 @@ RRDSET *rrdset_create_custom(
// could not use the name, use the id
rrdset_set_name(st, id);
st->title = strdupz(title);
st->state->old_title = strdupz(st->title);
json_fix_string(st->title);
st->title = rrd_string_strdupz(title);
st->rrdfamily = rrdfamily_create(host, st->family);
st->rrdfamily = rrdfamily_create(host, rrdset_family(st));
st->next = host->rrdset_root;
host->rrdset_root = st;
DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->rrdset_root, st, prev, next);
if(host->health_enabled) {
rrdsetvar_create(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT);
@ -793,9 +742,7 @@ RRDSET *rrdset_create_custom(
rrdsetvar_create(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, RRDVAR_OPTION_DEFAULT);
}
if(unlikely(rrdset_index_add(host, st) != st))
error("RRDSET: INTERNAL ERROR: attempt to index duplicate chart '%s'", st->id);
rrdset_index_add(host, st);
rrdsetcalc_link_matching(st);
rrdcalctemplate_link_matching(st);
@ -870,7 +817,7 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
// oops! the database is in the future
#ifdef NETDATA_INTERNAL_CHECKS
info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
" secs in the future (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
" secs in the future (counter #%zu, update #%zu). Adjusting it to current time.", rrdset_id(st), rrdhost_hostname(st->rrdhost), (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
#endif
st->last_collected_time.tv_sec = now.tv_sec - st->update_every;
@ -890,7 +837,7 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
// oops! the database is too far behind
#ifdef NETDATA_INTERNAL_CHECKS
info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
" secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
" secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", rrdset_id(st), rrdhost_hostname(st->rrdhost), (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
#endif
microseconds = (usec_t)since_last_usec;
@ -925,11 +872,11 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
}
#ifdef NETDATA_INTERNAL_CHECKS
debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds);
debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", rrdset_name(st), microseconds);
rrdset_debug(st, "NEXT: %llu microseconds", microseconds);
if(discarded && discarded != microseconds)
info("host '%s', chart '%s': discarded data collection time of %llu usec, replaced with %llu usec, reason: '%s'", st->rrdhost->hostname, st->id, discarded, microseconds, discard_reason?discard_reason:"UNDEFINED");
info("host '%s', chart '%s': discarded data collection time of %llu usec, replaced with %llu usec, reason: '%s'", rrdhost_hostname(st->rrdhost), rrdset_id(st), discarded, microseconds, discard_reason?discard_reason:"UNDEFINED");
#endif
@ -1081,6 +1028,8 @@ static void store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
t->last_collected_ut = point_end_time_ut;
store_metric_at_tier(rd, t, sp, point_end_time_ut);
}
rrdcontext_collected_rrddim(rd);
}
static inline size_t rrdset_done_interpolate(
@ -1114,7 +1063,7 @@ static inline size_t rrdset_done_interpolate(
for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) {
#ifdef NETDATA_INTERNAL_CHECKS
if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", rrdset_name(st), first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
#endif
@ -1140,7 +1089,7 @@ static inline size_t rrdset_done_interpolate(
NETDATA_DOUBLE_FORMAT
" * (%llu - %llu)"
" / (%llu - %llu)"
, rd->name
, rrddim_name(rd)
, new_value
, rd->calculated_value
, next_store_ut, last_collect_ut
@ -1157,7 +1106,7 @@ static inline size_t rrdset_done_interpolate(
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " NETDATA_DOUBLE_FORMAT " - EXTRAPOLATING",
rd->name
rrddim_name(rd)
, (NETDATA_DOUBLE)(next_store_ut - last_stored_ut)
);
#endif
@ -1193,7 +1142,7 @@ static inline size_t rrdset_done_interpolate(
rrdset_debug(st, "%s: CALC2 DEF " NETDATA_DOUBLE_FORMAT " = ((("
"(" NETDATA_DOUBLE_FORMAT " - " NETDATA_DOUBLE_FORMAT ")"
" * %llu"
" / %llu) + " NETDATA_DOUBLE_FORMAT, rd->name
" / %llu) + " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
, new_value
, rd->calculated_value, rd->last_calculated_value
, (next_store_ut - first_ut)
@ -1225,7 +1174,7 @@ static inline size_t rrdset_done_interpolate(
(void) ml_is_anomalous(rd, 0, false);
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rd->name, current_entry);
rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rrddim_name(rd), current_entry);
#endif
store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
@ -1275,7 +1224,7 @@ static inline void rrdset_done_fill_the_gap(RRDSET *st) {
current_entry = ((current_entry + 1) >= entries) ? 0 : current_entry + 1;
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING (FILLED THE GAP)", rd->name, current_entry);
rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING (FILLED THE GAP)", rrddim_name(rd), current_entry);
#endif
}
}
@ -1294,8 +1243,7 @@ static inline void rrdset_done_fill_the_gap(RRDSET *st) {
void rrdset_done(RRDSET *st) {
if(unlikely(netdata_exit)) return;
debug(D_RRD_CALLS, "rrdset_done() for chart %s", st->name);
rrdcontext_collected_rrdset(st);
debug(D_RRD_CALLS, "rrdset_done() for chart %s", rrdset_name(st));
RRDDIM *rd;
@ -1326,7 +1274,7 @@ void rrdset_done(RRDSET *st) {
#endif
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) {
error("Chart '%s' has the OBSOLETE flag set, but it is collected.", st->id);
error("Chart '%s' has the OBSOLETE flag set, but it is collected.", rrdset_id(st));
rrdset_isnot_obsolete(st);
}
@ -1334,7 +1282,7 @@ void rrdset_done(RRDSET *st) {
if(unlikely(st->usec_since_last_update > st->entries * update_every_ut &&
st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && st->rrd_memory_mode != RRD_MEMORY_MODE_NONE)) {
info("host '%s', chart %s: took too long to be updated (counter #%zu, update #%zu, %0.3" NETDATA_DOUBLE_MODIFIER
" secs). Resetting it.", st->rrdhost->hostname, st->name, st->counter, st->counter_done, (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);
" secs). Resetting it.", rrdhost_hostname(st->rrdhost), rrdset_name(st), st->counter, st->counter_done, (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);
rrdset_reset(st);
st->usec_since_last_update = update_every_ut;
store_this_entry = 0;
@ -1382,7 +1330,7 @@ void rrdset_done(RRDSET *st) {
info(
"%s: too old data (last updated at %"PRId64".%"PRId64", last collected at %"PRId64".%"PRId64"). "
"Resetting it. Will not store the next entry.",
st->name,
rrdset_name(st),
(int64_t)st->last_updated.tv_sec,
(int64_t)st->last_updated.tv_usec,
(int64_t)st->last_collected_time.tv_sec,
@ -1404,7 +1352,7 @@ void rrdset_done(RRDSET *st) {
info(
"%s: too old data (last updated at %" PRId64 ".%" PRId64 ", last collected at %" PRId64 ".%" PRId64 "). "
"Resetting it. Will not store the next entry.",
st->name,
rrdset_name(st),
(int64_t)st->last_updated.tv_sec,
(int64_t)st->last_updated.tv_usec,
(int64_t)st->last_collected_time.tv_sec,
@ -1467,9 +1415,9 @@ after_first_database_work:
if(unlikely(st->rrdhost->rrdpush_send_enabled))
rrdset_done_push(st);
if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE)) {
if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE))
goto after_second_database_work;
}
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "last_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last collection time)", (NETDATA_DOUBLE)last_collect_ut/USEC_PER_SEC);
@ -1506,7 +1454,7 @@ after_first_database_work:
}
if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) {
error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rd->name, st->id);
error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
rrddim_isnot_obsolete(st, rd);
}
@ -1515,11 +1463,12 @@ after_first_database_work:
" last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
, rd->calculated_value
" calculated_value = " NETDATA_DOUBLE_FORMAT
, rrddim_name(rd)
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
, rd->calculated_value
);
#endif
@ -1533,7 +1482,7 @@ after_first_database_work:
rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN " NETDATA_DOUBLE_FORMAT " = "
COLLECTED_NUMBER_FORMAT
" * " NETDATA_DOUBLE_FORMAT
" / " NETDATA_DOUBLE_FORMAT, rd->name
" / " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
, rd->calculated_value
, rd->collected_value
, (NETDATA_DOUBLE)rd->multiplier
@ -1558,7 +1507,7 @@ after_first_database_work:
rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100"
" * " COLLECTED_NUMBER_FORMAT
" / " COLLECTED_NUMBER_FORMAT
, rd->name
, rrddim_name(rd)
, rd->calculated_value
, rd->collected_value
, st->collected_total
@ -1579,7 +1528,7 @@ after_first_database_work:
// produces wrong results as far as incremental counters are concerned.
if(unlikely((uint64_t)rd->last_collected_value > (uint64_t)rd->collected_value)) {
debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
, st->name, rd->name
, rrdset_name(st), rrddim_name(rd)
, rd->last_collected_value
, rd->collected_value);
@ -1627,7 +1576,7 @@ after_first_database_work:
COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
")"
" * " NETDATA_DOUBLE_FORMAT
" / " NETDATA_DOUBLE_FORMAT, rd->name
" / " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
, rd->calculated_value
, rd->collected_value, rd->last_collected_value
, (NETDATA_DOUBLE)rd->multiplier
@ -1647,7 +1596,7 @@ after_first_database_work:
// to reset the calculation (it will give zero as the calculation for this second)
if(unlikely(rd->last_collected_value > rd->collected_value)) {
debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
, st->name, rd->name
, rrdset_name(st), rrddim_name(rd)
, rd->last_collected_value
, rd->collected_value
);
@ -1672,7 +1621,7 @@ after_first_database_work:
rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100"
" * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
" / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
, rd->name
, rrddim_name(rd)
, rd->calculated_value
, rd->collected_value, rd->last_collected_value
, st->collected_total, st->last_collected_total
@ -1688,7 +1637,7 @@ after_first_database_work:
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC " NETDATA_DOUBLE_FORMAT " = 0"
, rd->name
, rrddim_name(rd)
, rd->calculated_value
);
#endif
@ -1701,7 +1650,7 @@ after_first_database_work:
" last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
" calculated_value = " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
@ -1718,7 +1667,7 @@ after_first_database_work:
// if(unlikely(now_collect_ut < next_store_ut && st->counter_done > 1)) {
// // this is collected in the same interpolation point
// rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT");
// info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, st->name, st->counter_done, next_store_ut - now_collect_ut);
// info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, rrdset_name(st), st->counter_done, next_store_ut - now_collect_ut);
// }
// #endif
@ -1738,6 +1687,7 @@ after_second_database_work:
#ifdef ENABLE_ACLK
time_t mark = now_realtime_sec();
#endif
rrddim_foreach_read(rd, st) {
if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
continue;
@ -1752,7 +1702,7 @@ after_second_database_work:
continue;
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rd->name, rd->last_collected_value, rd->collected_value);
rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rrddim_name(rd), rd->last_collected_value, rd->collected_value);
#endif
rd->last_collected_value = rd->collected_value;
@ -1762,7 +1712,7 @@ after_second_database_work:
if(unlikely(!first_entry)) {
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT
") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rrddim_name(rd), rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
#endif
rd->last_calculated_value += rd->calculated_value;
@ -1779,7 +1729,7 @@ after_second_database_work:
case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT
") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value);
") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rrddim_name(rd), rd->last_calculated_value, rd->calculated_value);
#endif
rd->last_calculated_value = rd->calculated_value;
@ -1795,7 +1745,7 @@ after_second_database_work:
" last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
" calculated_value = " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
@ -1835,7 +1785,7 @@ after_second_database_work:
for( rd = st->dimensions, last = NULL ; likely(rd) ; ) {
if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && !rrddim_flag_check(rd, RRDDIM_FLAG_ACLK)
&& (rd->last_collected_time.tv_sec + rrdset_free_obsolete_time < now))) {
info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rrddim_name(rd), rrddim_id(rd), rrdset_name(st), rrdset_id(st));
const char *cache_filename = rrddim_cache_filename(rd);
if(cache_filename) {
@ -1900,8 +1850,9 @@ after_second_database_work:
}
}
rrdset_unlock(st);
rrdcontext_collected_rrdset(st);
rrdset_unlock(st);
netdata_thread_enable_cancelability();
}
@ -2047,8 +1998,8 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo
info("Initializing file '%s'.", fullfilename);
memset(st_on_file, 0, size);
}
else if(strncmp(st_on_file->id, st->id, RRD_ID_LENGTH_MAX_V019) != 0) {
error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, st->id);
else if(strncmp(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019) != 0) {
error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, rrdset_id(st));
memset(st_on_file, 0, size);
}
else if(st_on_file->memsize != size || st_on_file->entries != st->entries) {
@ -2094,7 +2045,7 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo
memset(st_on_file, 0, size);
// set the values we need
strncpyz(st_on_file->id, st->id, RRD_ID_LENGTH_MAX_V019 + 1);
strncpyz(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019 + 1);
strcpy(st_on_file->cache_filename, fullfilename);
strcpy(st_on_file->magic, RRDSET_MAGIC_V019);
st_on_file->memsize = size;

74
database/rrdsetvar.c
View File

@ -13,31 +13,31 @@ static inline void rrdsetvar_free_variables(RRDSETVAR *rs) {
// ------------------------------------------------------------------------
// CHART
rrdvar_free(host, &st->rrdvar_root_index, rs->var_local);
rrdvar_free(host, st->rrdvar_root_index, rs->var_local);
rs->var_local = NULL;
// ------------------------------------------------------------------------
// FAMILY
rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family);
rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family);
rs->var_family = NULL;
rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rs->var_family_name);
rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_name);
rs->var_family_name = NULL;
// ------------------------------------------------------------------------
// HOST
rrdvar_free(host, &host->rrdvar_root_index, rs->var_host);
rrdvar_free(host, host->rrdvar_root_index, rs->var_host);
rs->var_host = NULL;
rrdvar_free(host, &host->rrdvar_root_index, rs->var_host_name);
rrdvar_free(host, host->rrdvar_root_index, rs->var_host_name);
rs->var_host_name = NULL;
// ------------------------------------------------------------------------
// KEYS
freez(rs->key_fullid);
string_freez(rs->key_fullid);
rs->key_fullid = NULL;
freez(rs->key_fullname);
string_freez(rs->key_fullname);
rs->key_fullname = NULL;
}
@ -58,40 +58,38 @@ static inline void rrdsetvar_create_variables(RRDSETVAR *rs) {
// KEYS
char buffer[RRDVAR_MAX_LENGTH + 1];
snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rs->variable);
rs->key_fullid = strdupz(buffer);
snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), string2str(rs->variable));
rs->key_fullid = string_strdupz(buffer);
snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable);
rs->key_fullname = strdupz(buffer);
snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), string2str(rs->variable));
rs->key_fullname = string_strdupz(buffer);
// ------------------------------------------------------------------------
// CHART
rs->var_local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->variable, rs->type, options, rs->value);
rs->var_local = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->variable, rs->type, options, rs->value);
// ------------------------------------------------------------------------
// FAMILY
rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value);
rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value);
rs->var_family = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value);
rs->var_family_name = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value);
// ------------------------------------------------------------------------
// HOST
rs->var_host = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value);
rs->var_host_name = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value);
rs->var_host = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value);
rs->var_host_name = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value);
}
RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options) {
debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable);
debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", rrdset_id(st), rrdset_name(st), variable);
RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR));
rs->variable = strdupz(variable);
rs->hash = simple_hash(rs->variable);
rs->variable = string_strdupz(variable);
rs->type = type;
rs->value = value;
rs->options = options;
rs->rrdset = st;
rs->next = st->variables;
st->variables = rs;
DOUBLE_LINKED_LIST_PREPEND_UNSAFE(st->variables, rs, prev, next);
rrdsetvar_create_variables(rs);
@ -99,7 +97,7 @@ RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type,
}
void rrdsetvar_rename_all(RRDSET *st) {
debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name);
debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", rrdset_id(st), rrdset_name(st));
RRDSETVAR *rs;
for(rs = st->variables; rs ; rs = rs->next)
@ -110,21 +108,13 @@ void rrdsetvar_rename_all(RRDSET *st) {
void rrdsetvar_free(RRDSETVAR *rs) {
RRDSET *st = rs->rrdset;
debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable);
debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", rrdset_id(st), rrdset_name(st), string2str(rs->variable));
if(st->variables == rs) {
st->variables = rs->next;
}
else {
RRDSETVAR *t;
for (t = st->variables; t && t->next != rs; t = t->next);
if(!t) error("RRDSETVAR '%s' not found in chart '%s' variables linked list", rs->key_fullname, st->id);
else t->next = rs->next;
}
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->variables, rs, prev, next);
rrdsetvar_free_variables(rs);
freez(rs->variable);
string_freez(rs->variable);
if(rs->options & RRDVAR_OPTION_ALLOCATED)
freez(rs->value);
@ -138,24 +128,22 @@ void rrdsetvar_free(RRDSETVAR *rs) {
RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name) {
RRDHOST *host = st->rrdhost;
char *n = strdupz(name);
rrdvar_fix_name(n);
uint32_t hash = simple_hash(n);
STRING *n = rrdvar_name_to_string(name);
rrdset_wrlock(st);
// find it
RRDSETVAR *rs;
for(rs = st->variables; rs ; rs = rs->next) {
if(hash == rs->hash && strcmp(n, rs->variable) == 0) {
if(rs->variable == n) {
rrdset_unlock(st);
if(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) {
freez(n);
string_freez(n);
return rs;
}
else {
error("RRDSETVAR: custom variable '%s' on chart '%s' of host '%s', conflicts with an internal chart variable", n, st->id, host->hostname);
freez(n);
error("RRDSETVAR: custom variable '%s' on chart '%s' of host '%s', conflicts with an internal chart variable", string2str(n), rrdset_id(st), rrdhost_hostname(host));
string_freez(n);
return NULL;
}
}
@ -166,17 +154,17 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
NETDATA_DOUBLE *v = mallocz(sizeof(NETDATA_DOUBLE));
*v = NAN;
rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR);
rs = rrdsetvar_create(st, string2str(n), RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR);
rrdset_unlock(st);
freez(n);
string_freez(n);
return rs;
}
void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, NETDATA_DOUBLE value) {
if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) || !(rs->options & RRDVAR_OPTION_ALLOCATED)) {
error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " NETDATA_DOUBLE_FORMAT
" but the variable is not a custom chart one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value);
" but the variable is not a custom chart one.", string2str(rs->variable), rrdset_id(rs->rrdset), rrdhost_hostname(rs->rrdset->rrdhost), value);
}
else {
NETDATA_DOUBLE *v = rs->value;

8
database/rrdsetvar.h
View File

@ -12,11 +12,10 @@
// these variables
struct rrdsetvar {
char *variable; // variable name
uint32_t hash; // variable name hash
STRING *variable; // variable name
char *key_fullid; // chart type.chart id.variable
char *key_fullname; // chart type.chart name.variable
STRING *key_fullid; // chart type.chart id.variable
STRING *key_fullname; // chart type.chart name.variable
RRDVAR_TYPE type;
void *value;
@ -32,6 +31,7 @@ struct rrdsetvar {
struct rrdset *rrdset;
struct rrdsetvar *next;
struct rrdsetvar *prev;
};
extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name);

150
database/rrdvar.c
View File

@ -20,87 +20,74 @@ inline int rrdvar_fix_name(char *variable) {
return fixed;
}
int rrdvar_compare(void* a, void* b) {
if(((RRDVAR *)a)->hash < ((RRDVAR *)b)->hash) return -1;
else if(((RRDVAR *)a)->hash > ((RRDVAR *)b)->hash) return 1;
else return strcmp(((RRDVAR *)a)->name, ((RRDVAR *)b)->name);
static inline RRDVAR *rrdvar_index_add(DICTIONARY *dict, RRDVAR *rv) {
return dictionary_set(dict, rrdvar_name(rv), rv, sizeof(RRDVAR));
}
static inline RRDVAR *rrdvar_index_add(avl_tree_lock *tree, RRDVAR *rv) {
RRDVAR *ret = (RRDVAR *)avl_insert_lock(tree, (avl_t *)(rv));
if(ret != rv)
debug(D_VARIABLES, "Request to insert RRDVAR '%s' into index failed. Already exists.", rv->name);
static inline RRDVAR *rrdvar_index_del(DICTIONARY *dict, RRDVAR *rv) {
if(dictionary_del(dict, rrdvar_name(rv)) != 0) {
error("Request to remove RRDVAR '%s' from index failed. Not Found.", rrdvar_name(rv));
return NULL;
}
return ret;
return rv;
}
static inline RRDVAR *rrdvar_index_del(avl_tree_lock *tree, RRDVAR *rv) {
RRDVAR *ret = (RRDVAR *)avl_remove_lock(tree, (avl_t *)(rv));
if(!ret)
error("Request to remove RRDVAR '%s' from index failed. Not Found.", rv->name);
return ret;
static inline RRDVAR *rrdvar_index_find(DICTIONARY *dict, STRING *name) {
return dictionary_get(dict, string2str(name));
}
static inline RRDVAR *rrdvar_index_find(avl_tree_lock *tree, const char *name, uint32_t hash) {
RRDVAR tmp;
tmp.name = (char *)name;
tmp.hash = (hash)?hash:simple_hash(tmp.name);
return (RRDVAR *)avl_search_lock(tree, (avl_t *)&tmp);
}
inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) {
inline void rrdvar_free(RRDHOST *host, DICTIONARY *dict, RRDVAR *rv) {
(void)host;
if(!rv) return;
if(tree) {
debug(D_VARIABLES, "Deleting variable '%s'", rv->name);
if(unlikely(!rrdvar_index_del(tree, rv)))
error("RRDVAR: Attempted to delete variable '%s' from host '%s', but it is not found.", rv->name, host->hostname);
if(dict) {
debug(D_VARIABLES, "Deleting variable '%s'", rrdvar_name(rv));
if(unlikely(!rrdvar_index_del(dict, rv)))
error("RRDVAR: Attempted to delete variable '%s' from host '%s', but it is not found.", rrdvar_name(rv), rrdhost_hostname(host));
}
if(rv->options & RRDVAR_OPTION_ALLOCATED)
freez(rv->value);
freez(rv->name);
string_freez(rv->name);
freez(rv);
}
inline RRDVAR *rrdvar_create_and_index(const char *scope __maybe_unused, avl_tree_lock *tree, const char *name,
RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value) {
inline STRING *rrdvar_name_to_string(const char *name) {
char *variable = strdupz(name);
rrdvar_fix_name(variable);
uint32_t hash = simple_hash(variable);
STRING *name_string = string_strdupz(variable);
freez(variable);
return name_string;
}
RRDVAR *rv = rrdvar_index_find(tree, variable, hash);
inline RRDVAR *rrdvar_create_and_index(const char *scope __maybe_unused, DICTIONARY *dict, STRING *name,
RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value) {
RRDVAR *rv = rrdvar_index_find(dict, name);
if(unlikely(!rv)) {
debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", variable, scope);
debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", string2str(name), scope);
rv = callocz(1, sizeof(RRDVAR));
rv->name = variable;
rv->hash = hash;
rv->name = string_dup(name);
rv->type = type;
rv->options = options;
rv->value = value;
rv->last_updated = now_realtime_sec();
RRDVAR *ret = rrdvar_index_add(tree, rv);
RRDVAR *ret = rrdvar_index_add(dict, rv);
if(unlikely(ret != rv)) {
debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", variable, scope);
debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", string2str(name), scope);
freez(rv);
freez(variable);
rv = NULL;
}
else
debug(D_VARIABLES, "Variable '%s' created in scope '%s'", variable, scope);
debug(D_VARIABLES, "Variable '%s' created in scope '%s'", string2str(name), scope);
}
else {
debug(D_VARIABLES, "Variable '%s' is already found in scope '%s'.", variable, scope);
// already exists
freez(variable);
debug(D_VARIABLES, "Variable '%s' is already found in scope '%s'.", string2str(name), scope);
// this is important
// it must return NULL - not the existing variable - or double-free will happen
@ -110,57 +97,48 @@ inline RRDVAR *rrdvar_create_and_index(const char *scope __maybe_unused, avl_tre
return rv;
}
void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock) {
// This is not bullet proof - avl should support some means to destroy it
// with a callback for each item already in the index
RRDVAR *rv, *last = NULL;
while((rv = (RRDVAR *)tree_lock->avl_tree.root)) {
if(unlikely(rv == last)) {
error("RRDVAR: INTERNAL ERROR: Cannot cleanup tree of RRDVARs");
break;
}
last = rv;
rrdvar_free(host, tree_lock, rv);
void rrdvar_free_remaining_variables(RRDHOST *host, DICTIONARY *dict) {
RRDVAR *rv;
dfe_start_rw(dict, rv, DICTIONARY_LOCK_REENTRANT) {
rrdvar_free(host, dict, rv);
}
dfe_done(rv);
}
// ----------------------------------------------------------------------------
// CUSTOM HOST VARIABLES
inline int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void * /*rrdvar*/, void * /*data*/), void *data) {
return avl_traverse_lock(&host->rrdvar_root_index, callback, data);
inline int rrdvar_walkthrough_read(DICTIONARY *dict, int (*callback)(const char *name, void *rrdvar, void *data), void *data) {
return dictionary_walkthrough_read(dict, callback, data);
}
static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *tree_lock, const char *name) {
static RRDVAR *rrdvar_custom_variable_create(const char *scope, DICTIONARY *dict, const char *name) {
NETDATA_DOUBLE *v = callocz(1, sizeof(NETDATA_DOUBLE));
*v = NAN;
RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v);
STRING *name_string = rrdvar_name_to_string(name);
RRDVAR *rv = rrdvar_create_and_index(scope, dict, name_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v);
if(unlikely(!rv)) {
freez(v);
debug(D_VARIABLES, "Requested variable '%s' already exists - possibly 2 plugins are updating it at the same time.", name);
char *variable = strdupz(name);
rrdvar_fix_name(variable);
uint32_t hash = simple_hash(variable);
debug(D_VARIABLES, "Requested variable '%s' already exists - possibly 2 plugins are updating it at the same time.", string2str(name_string));
// find the existing one to return it
rv = rrdvar_index_find(tree_lock, variable, hash);
freez(variable);
rv = rrdvar_index_find(dict, name_string);
}
string_freez(name_string);
return rv;
}
RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) {
return rrdvar_custom_variable_create("host", &host->rrdvar_root_index, name);
return rrdvar_custom_variable_create("host", host->rrdvar_root_index, name);
}
void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value) {
if(rv->type != RRDVAR_TYPE_CALCULATED || !(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR) || !(rv->options & RRDVAR_OPTION_ALLOCATED))
error("requested to set variable '%s' to value " NETDATA_DOUBLE_FORMAT " but the variable is not a custom one.", rv->name, value);
error("requested to set variable '%s' to value " NETDATA_DOUBLE_FORMAT " but the variable is not a custom one.", rrdvar_name(rv), value);
else {
NETDATA_DOUBLE *v = rv->value;
if(*v != value) {
@ -174,10 +152,6 @@ void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE v
}
}
int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR * /*rv*/, void * /*data*/), void *data) {
return avl_traverse_lock(&host->rrdvar_root_index, (int (*)(void *, void *))callback, data);
}
// ----------------------------------------------------------------------------
// RRDVAR lookup
@ -214,26 +188,26 @@ NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) {
}
}
int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, NETDATA_DOUBLE *result) {
int health_variable_lookup(STRING *variable, RRDCALC *rc, NETDATA_DOUBLE *result) {
RRDSET *st = rc->rrdset;
if(!st) return 0;
RRDHOST *host = st->rrdhost;
RRDVAR *rv;
rv = rrdvar_index_find(&st->rrdvar_root_index, variable, hash);
rv = rrdvar_index_find(st->rrdvar_root_index, variable);
if(rv) {
*result = rrdvar2number(rv);
return 1;
}
rv = rrdvar_index_find(&st->rrdfamily->rrdvar_root_index, variable, hash);
rv = rrdvar_index_find(st->rrdfamily->rrdvar_root_index, variable);
if(rv) {
*result = rrdvar2number(rv);
return 1;
}
rv = rrdvar_index_find(&host->rrdvar_root_index, variable, hash);
rv = rrdvar_index_find(host->rrdvar_root_index, variable);
if(rv) {
*result = rrdvar2number(rv);
return 1;
@ -251,16 +225,16 @@ struct variable2json_helper {
RRDVAR_OPTIONS options;
};
static int single_variable2json(void *entry, void *data) {
static int single_variable2json(const char *name __maybe_unused, void *entry, void *data) {
struct variable2json_helper *helper = (struct variable2json_helper *)data;
RRDVAR *rv = (RRDVAR *)entry;
NETDATA_DOUBLE value = rrdvar2number(rv);
if (helper->options == RRDVAR_OPTION_DEFAULT || rv->options & helper->options) {
if(unlikely(isnan(value) || isinf(value)))
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rv->name);
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rrdvar_name(rv));
else
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" NETDATA_DOUBLE_MODIFIER, helper->counter?",":"", rv->name, (NETDATA_DOUBLE)value);
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" NETDATA_DOUBLE_MODIFIER, helper->counter?",":"", rrdvar_name(rv), (NETDATA_DOUBLE)value);
helper->counter++;
}
@ -276,7 +250,7 @@ void health_api_v1_chart_custom_variables2json(RRDSET *st, BUFFER *buf) {
};
buffer_sprintf(buf, "{");
avl_traverse_lock(&st->rrdvar_root_index, single_variable2json, (void *)&helper);
rrdvar_walkthrough_read(st->rrdvar_root_index, single_variable2json, &helper);
buffer_strcat(buf, "\n\t\t\t}");
}
@ -289,16 +263,16 @@ void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf) {
.options = RRDVAR_OPTION_DEFAULT
};
buffer_sprintf(buf, "{\n\t\"chart\": \"%s\",\n\t\"chart_name\": \"%s\",\n\t\"chart_context\": \"%s\",\n\t\"chart_variables\": {", st->id, st->name, st->context);
avl_traverse_lock(&st->rrdvar_root_index, single_variable2json, (void *)&helper);
buffer_sprintf(buf, "{\n\t\"chart\": \"%s\",\n\t\"chart_name\": \"%s\",\n\t\"chart_context\": \"%s\",\n\t\"chart_variables\": {", rrdset_id(st), rrdset_name(st), rrdset_context(st));
rrdvar_walkthrough_read(st->rrdvar_root_index, single_variable2json, &helper);
buffer_sprintf(buf, "\n\t},\n\t\"family\": \"%s\",\n\t\"family_variables\": {", st->family);
buffer_sprintf(buf, "\n\t},\n\t\"family\": \"%s\",\n\t\"family_variables\": {", rrdset_family(st));
helper.counter = 0;
avl_traverse_lock(&st->rrdfamily->rrdvar_root_index, single_variable2json, (void *)&helper);
rrdvar_walkthrough_read(st->rrdfamily->rrdvar_root_index, single_variable2json, &helper);
buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",\n\t\"host_variables\": {", host->hostname);
buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",\n\t\"host_variables\": {", rrdhost_hostname(host));
helper.counter = 0;
avl_traverse_lock(&host->rrdvar_root_index, single_variable2json, (void *)&helper);
rrdvar_walkthrough_read(host->rrdvar_root_index, single_variable2json, &helper);
buffer_strcat(buf, "\n\t}\n}\n");
}

27
database/rrdvar.h
View File

@ -5,8 +5,6 @@
#include "libnetdata/libnetdata.h"
extern int rrdvar_compare(void *a, void *b);
typedef enum rrdvar_type {
RRDVAR_TYPE_CALCULATED = 1,
RRDVAR_TYPE_TIME_T = 2,
@ -32,35 +30,34 @@ typedef enum rrdvar_options {
// 2. at each context (RRDFAMILY.rrdvar_root_index)
// 3. at each host (RRDHOST.rrdvar_root_index)
struct rrdvar {
avl_t avl;
char *name;
uint32_t hash;
RRDVAR_TYPE type;
RRDVAR_OPTIONS options;
STRING *name;
void *value;
time_t last_updated;
RRDVAR_OPTIONS options:16;
RRDVAR_TYPE type:8;
};
#define rrdvar_name(rv) string2str((rv)->name)
#define RRDVAR_MAX_LENGTH 1024
extern int rrdvar_fix_name(char *variable);
#include "rrd.h"
extern STRING *rrdvar_name_to_string(const char *name);
extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name);
extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value);
extern int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data);
extern void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock);
extern void rrdvar_free_remaining_variables(RRDHOST *host, DICTIONARY *dict);
extern int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data);
extern int rrdvar_walkthrough_read(DICTIONARY *dict, int (*callback)(const char *name, void *rrdvar, void *data), void *data);
extern NETDATA_DOUBLE rrdvar2number(RRDVAR *rv);
extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value);
extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv);
extern RRDVAR *rrdvar_create_and_index(const char *scope, DICTIONARY *dict, STRING *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value);
extern void rrdvar_free(RRDHOST *host, DICTIONARY *dict, RRDVAR *rv);
#endif //NETDATA_RRDVAR_H

26
database/sqlite/sqlite_aclk.c
View File

@ -237,15 +237,14 @@ struct aclk_database_worker_config *find_inactive_wc_by_node_id(char *node_id)
void aclk_sync_exit_all()
{
rrd_rdlock();
RRDHOST *host = localhost;
while(host) {
RRDHOST *host;
rrdhost_foreach_read(host) {
struct aclk_database_worker_config *wc = host->dbsync_worker;
if (wc) {
wc->is_shutting_down = 1;
(void) aclk_database_deq_cmd(wc);
uv_cond_signal(&wc->cmd_cond);
}
host = host->next;
}
rrd_unlock();
@ -320,7 +319,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
char node_str[UUID_STR_LEN] = "<none>";
if (likely(host->node_id))
uuid_unparse_lower(*host->node_id, node_str);
internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", host->hostname, host->machine_guid, node_str);
internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", rrdhost_hostname(host), host->machine_guid, node_str);
#endif
return 0;
}
@ -335,7 +334,7 @@ int aclk_start_sync_thread(void *data, int argc, char **argv, char **column)
uuid_unparse_lower(*((uuid_t *) argv[0]), uuid_str);
RRDHOST *host = rrdhost_find_by_guid(uuid_str, 0);
RRDHOST *host = rrdhost_find_by_guid(uuid_str);
if (host == localhost)
return 0;
@ -526,7 +525,7 @@ void aclk_database_worker(void *arg)
char threadname[NETDATA_THREAD_NAME_MAX+1];
if (wc->host)
snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->host->hostname);
snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", rrdhost_hostname(wc->host));
else {
snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->uuid_str);
threadname[11] = '\0';
@ -705,14 +704,14 @@ void aclk_database_worker(void *arg)
case ACLK_DATABASE_TIMER:
if (unlikely(localhost && !wc->host && !wc->is_orphan)) {
if (claimed()) {
wc->host = rrdhost_find_by_guid(wc->host_guid, 0);
wc->host = rrdhost_find_by_guid(wc->host_guid);
if (wc->host) {
info("HOST %s (%s) detected as active", wc->host->hostname, wc->host_guid);
snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->host->hostname);
info("HOST %s (%s) detected as active", rrdhost_hostname(wc->host), wc->host_guid);
snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", rrdhost_hostname(wc->host));
uv_thread_set_name_np(wc->thread, threadname);
wc->host->dbsync_worker = wc;
if (unlikely(!wc->hostname))
wc->hostname = strdupz(wc->host->hostname);
wc->hostname = strdupz(rrdhost_hostname(wc->host));
aclk_del_worker_thread(wc);
wc->node_info_send = 1;
}
@ -844,7 +843,7 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
uuid_unparse_lower(*node_id, wc->node_id);
if (likely(host)) {
host->dbsync_worker = (void *)wc;
wc->hostname = strdupz(host->hostname);
wc->hostname = strdupz(rrdhost_hostname(host));
}
else
wc->hostname = get_hostname_by_node_id(wc->node_id);
@ -1039,12 +1038,11 @@ void aclk_data_rotated(void)
time_t next_rotation_time = now_realtime_sec()+ACLK_DATABASE_ROTATION_DELAY;
rrd_rdlock();
RRDHOST *this_host = localhost;
while (this_host) {
RRDHOST *this_host;
rrdhost_foreach_read(this_host) {
struct aclk_database_worker_config *wc = this_host->dbsync_worker;
if (wc)
wc->rotation_after = next_rotation_time;
this_host = this_host->next;
}
rrd_unlock();

16
database/sqlite/sqlite_aclk.h
View File

@ -216,13 +216,17 @@ static inline RRDHOST *find_host_by_node_id(char *node_id)
if (uuid_parse(node_id, node_uuid))
return NULL;
RRDHOST *host = localhost;
while(host) {
if (host->node_id && !(uuid_compare(*host->node_id, node_uuid)))
return host;
host = host->next;
rrd_rdlock();
RRDHOST *host, *ret = NULL;
rrdhost_foreach_read(host) {
if (host->node_id && !(uuid_compare(*host->node_id, node_uuid))) {
ret = host;
break;
}
}
return NULL;
rrd_unlock();
return ret;
}

56
database/sqlite/sqlite_aclk_alert.c
View File

@ -220,7 +220,7 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
int rc;
if (unlikely(!wc->alert_updates)) {
log_access("ACLK STA [%s (%s)]: Ignoring alert push event, updates have been turned off for this node.", wc->node_id, wc->host ? wc->host->hostname : "N/A");
log_access("ACLK STA [%s (%s)]: Ignoring alert push event, updates have been turned off for this node.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
return;
}
@ -300,9 +300,9 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
alarm_log.config_hash = strdupz((char *)uuid_str);
alarm_log.utc_offset = wc->host->utc_offset;
alarm_log.timezone = strdupz((char *)wc->host->abbrev_timezone);
alarm_log.timezone = strdupz(rrdhost_abbrev_timezone(wc->host));
alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) :
strdupz((char *)wc->host->health_default_exec);
strdupz((char *)string2str(wc->host->health_default_exec));
alarm_log.conf_source = strdupz((char *)sqlite3_column_text(res, 16));
char *edit_command = sqlite3_column_bytes(res, 16) > 0 ?
@ -374,7 +374,7 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
log_access(
"ACLK RES [%s (%s)]: ALERTS SENT from %" PRIu64 " to %" PRIu64 " batch=%" PRIu64,
wc->node_id,
wc->host ? wc->host->hostname : "N/A",
wc->host ? rrdhost_hostname(wc->host) : "N/A",
log_first_sequence_id,
log_last_sequence_id,
wc->alerts_batch_id);
@ -424,9 +424,7 @@ void aclk_send_alarm_health_log(char *node_id)
struct aclk_database_worker_config *wc = find_inactive_wc_by_node_id(node_id);
if (likely(!wc)) {
rrd_rdlock();
RRDHOST *host = find_host_by_node_id(node_id);
rrd_unlock();
if (likely(host))
wc = (struct aclk_database_worker_config *)host->dbsync_worker;
}
@ -460,9 +458,7 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
RRDHOST *host = wc->host;
if (unlikely(!host)) {
rrd_rdlock();
host = find_host_by_node_id(wc->node_id);
rrd_unlock();
if (unlikely(!host)) {
log_access(
@ -554,7 +550,7 @@ void aclk_send_alarm_configuration(char *config_hash)
return;
}
log_access("ACLK REQ [%s (%s)]: Request to send alert config %s.", wc->node_id, wc->host ? wc->host->hostname : "N/A", config_hash);
log_access("ACLK REQ [%s (%s)]: Request to send alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
struct aclk_database_cmd cmd;
memset(&cmd, 0, sizeof(cmd));
@ -664,14 +660,14 @@ int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct
}
if (likely(p_alarm_config.cfg_hash)) {
log_access("ACLK RES [%s (%s)]: Sent alert config %s.", wc->node_id, wc->host ? wc->host->hostname : "N/A", config_hash);
log_access("ACLK RES [%s (%s)]: Sent alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
aclk_send_provide_alarm_cfg(&p_alarm_config);
freez((char *) cmd.data_param);
freez(p_alarm_config.cfg_hash);
destroy_aclk_alarm_configuration(&alarm_config);
}
else
log_access("ACLK STA [%s (%s)]: Alert config for %s not found.", wc->node_id, wc->host ? wc->host->hostname : "N/A", config_hash);
log_access("ACLK STA [%s (%s)]: Alert config for %s not found.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
bind_fail:
rc = sqlite3_finalize(res);
@ -697,9 +693,7 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start
return;
struct aclk_database_worker_config *wc = NULL;
rrd_rdlock();
RRDHOST *host = find_host_by_node_id(node_id);
rrd_unlock();
if (likely(host)) {
wc = (struct aclk_database_worker_config *)host->dbsync_worker ?
(struct aclk_database_worker_config *)host->dbsync_worker :
@ -716,7 +710,7 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start
wc = (struct aclk_database_worker_config *)find_inactive_wc_by_node_id(node_id);
if (likely(wc)) {
log_access("ACLK REQ [%s (%s)]: ALERTS STREAM from %"PRIu64" batch=%"PRIu64, node_id, wc->host ? wc->host->hostname : "N/A", start_seq_id, batch_id);
log_access("ACLK REQ [%s (%s)]: ALERTS STREAM from %"PRIu64" batch=%"PRIu64, node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", start_seq_id, batch_id);
__sync_synchronize();
wc->alerts_batch_id = batch_id;
wc->alerts_start_seq_id = start_seq_id;
@ -744,7 +738,7 @@ void sql_process_queue_removed_alerts_to_aclk(struct aclk_database_worker_config
db_execute(buffer_tostring(sql));
log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS", wc->node_id, wc->host ? wc->host->hostname : "N/A");
log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
buffer_free(sql);
@ -780,17 +774,15 @@ void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, uint64_t sn
return;
struct aclk_database_worker_config *wc = NULL;
rrd_rdlock();
RRDHOST *host = find_host_by_node_id(node_id);
if (likely(host))
wc = (struct aclk_database_worker_config *)host->dbsync_worker;
rrd_unlock();
if (likely(wc)) {
log_access(
"IN [%s (%s)]: Request to send alerts snapshot, snapshot_id %" PRIu64 " and ack_sequence_id %" PRIu64,
wc->node_id,
wc->host ? wc->host->hostname : "N/A",
wc->host ? rrdhost_hostname(wc->host) : "N/A",
snapshot_id,
sequence_id);
if (wc->alerts_snapshot_id == snapshot_id)
@ -831,13 +823,13 @@ void aclk_mark_alert_cloud_ack(char *uuid_str, uint64_t alerts_ack_sequence_id)
#ifdef ENABLE_ACLK
void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_ENTRY *ae, RRDHOST *host)
{
char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN");
char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");
char config_hash_id[GUID_LEN + 1];
uuid_unparse_lower(ae->config_hash_id, config_hash_id);
alarm_log->chart = strdupz((char *)ae->chart);
alarm_log->name = strdupz((char *)ae->name);
alarm_log->family = strdupz((char *)ae->family);
alarm_log->chart = strdupz(ae_chart_name(ae));
alarm_log->name = strdupz(ae_name(ae));
alarm_log->family = strdupz(ae_family(ae));
alarm_log->batch_id = 0;
alarm_log->sequence_id = 0;
@ -846,9 +838,9 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN
alarm_log->config_hash = strdupz((char *)config_hash_id);
alarm_log->utc_offset = host->utc_offset;
alarm_log->timezone = strdupz((char *)host->abbrev_timezone);
alarm_log->exec_path = ae->exec ? strdupz((char *)ae->exec) : strdupz((char *)host->health_default_exec);
alarm_log->conf_source = ae->source ? strdupz((char *)ae->source) : strdupz((char *)"");
alarm_log->timezone = strdupz(rrdhost_abbrev_timezone(host));
alarm_log->exec_path = ae->exec ? strdupz(ae_exec(ae)) : strdupz((char *)string2str(host->health_default_exec));
alarm_log->conf_source = ae->source ? strdupz(ae_source(ae)) : strdupz((char *)"");
alarm_log->command = strdupz((char *)edit_command);
@ -861,19 +853,19 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN
alarm_log->last_repeat = (time_t)ae->last_repeat;
alarm_log->silenced =
((ae->flags & HEALTH_ENTRY_FLAG_SILENCED) || (ae->recipient && !strncmp((char *)ae->recipient, "silent", 6))) ?
((ae->flags & HEALTH_ENTRY_FLAG_SILENCED) || (ae->recipient && !strncmp(ae_recipient(ae), "silent", 6))) ?
1 :
0;
alarm_log->value_string = strdupz(ae->new_value_string);
alarm_log->old_value_string = strdupz(ae->old_value_string);
alarm_log->value_string = strdupz(ae_new_value_string(ae));
alarm_log->old_value_string = strdupz(ae_old_value_string(ae));
alarm_log->value = (!isnan(ae->new_value)) ? (NETDATA_DOUBLE)ae->new_value : 0;
alarm_log->old_value = (!isnan(ae->old_value)) ? (NETDATA_DOUBLE)ae->old_value : 0;
alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
alarm_log->rendered_info = ae->info ? strdupz(ae->info) : strdupz((char *)"");
alarm_log->chart_context = ae->chart_context ? strdupz(ae->chart_context) : strdupz((char *)"");
alarm_log->rendered_info = strdupz(ae_info(ae));
alarm_log->chart_context = strdupz(ae_chart_context(ae));
freez(edit_command);
}
@ -905,7 +897,7 @@ void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, stru
UNUSED(cmd);
// we perhaps we don't need this for snapshots
if (unlikely(!wc->alert_updates)) {
log_access("ACLK STA [%s (%s)]: Ignoring alert snapshot event, updates have been turned off for this node.", wc->node_id, wc->host ? wc->host->hostname : "N/A");
log_access("ACLK STA [%s (%s)]: Ignoring alert snapshot event, updates have been turned off for this node.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
return;
}
@ -921,7 +913,7 @@ void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, stru
if (unlikely(!claim_id))
return;
log_access("ACLK REQ [%s (%s)]: Sending alerts snapshot, snapshot_id %" PRIu64, wc->node_id, wc->host ? wc->host->hostname : "N/A", wc->alerts_snapshot_id);
log_access("ACLK REQ [%s (%s)]: Sending alerts snapshot, snapshot_id %" PRIu64, wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", wc->alerts_snapshot_id);
aclk_mark_alert_cloud_ack(wc->uuid_str, wc->alerts_ack_sequence_id);

55
database/sqlite/sqlite_aclk_chart.c
View File

@ -153,10 +153,10 @@ int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_dat
chart_payload.config_hash = get_str_from_uuid(&st->state->hash_id);
chart_payload.update_every = st->update_every;
chart_payload.memory_mode = st->rrd_memory_mode;
chart_payload.name = (char *)st->name;
chart_payload.name = (char *)rrdset_name(st);
chart_payload.node_id = wc->node_id;
chart_payload.claim_id = claim_id;
chart_payload.id = strdupz(st->id);
chart_payload.id = strdupz(rrdset_id(st));
chart_payload.chart_labels = rrdlabels_create();
rrdlabels_copy(chart_payload.chart_labels, st->state->chart_labels);
@ -304,7 +304,7 @@ void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_d
log_access(
"ACLK STA [%s (%s)]: Ignoring chart push event, updates have been turned off for this node.",
wc->node_id,
wc->host ? wc->host->hostname : "N/A");
wc->host ? rrdhost_hostname(wc->host) : "N/A");
return;
}
@ -504,7 +504,7 @@ int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_d
log_access(
"ACLK REQ [%s (%s)]: Sending chart config for %s.",
wc->node_id,
wc->host ? wc->host->hostname : "N/A",
wc->host ? rrdhost_hostname(wc->host) : "N/A",
hash_id);
aclk_chart_config_updated(&chart_config, 1);
destroy_chart_config_updated(&chart_config);
@ -512,7 +512,7 @@ int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_d
log_access(
"ACLK STA [%s (%s)]: Chart config for %s not found.",
wc->node_id,
wc->host ? wc->host->hostname : "N/A",
wc->host ? rrdhost_hostname(wc->host) : "N/A",
hash_id);
bind_fail:
@ -552,7 +552,7 @@ void aclk_receive_chart_ack(struct aclk_database_worker_config *wc, struct aclk_
log_access(
"ACLK STA [%s (%s)]: CHARTS ACKNOWLEDGED IN THE DATABASE UP TO %" PRIu64,
wc->node_id,
wc->host ? wc->host->hostname : "N/A",
wc->host ? rrdhost_hostname(wc->host) : "N/A",
cmd.param1);
bind_fail:
@ -637,7 +637,7 @@ void aclk_get_chart_config(char **hash_id)
log_access(
"ACLK REQ [%s (%s)]: Request %d for chart config with hash %s received.",
wc->node_id,
wc->host ? wc->host->hostname : "N/A",
wc->host ? rrdhost_hostname(wc->host) : "N/A",
i,
hash_id[i]);
cmd.data_param = (void *)strdupz(hash_id[i]);
@ -660,11 +660,10 @@ static void aclk_submit_param_command(char *node_id, enum aclk_database_opcode a
cmd.opcode = aclk_command;
cmd.param1 = param;
rrd_rdlock();
RRDHOST *host = find_host_by_node_id(node_id);
if (likely(host))
wc = (struct aclk_database_worker_config *)host->dbsync_worker;
rrd_unlock();
if (wc)
aclk_database_enq_cmd(wc, &cmd);
else {
@ -702,8 +701,8 @@ void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at
struct aclk_database_worker_config *wc = find_inactive_wc_by_node_id(node_id);
rrd_rdlock();
RRDHOST *host = localhost;
while(host) {
RRDHOST *host;
rrdhost_foreach_read(host) {
if (wc || (host->node_id && !(uuid_compare(*host->node_id, node_uuid)))) {
rrd_unlock();
if (!wc)
@ -771,10 +770,8 @@ void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at
}
return;
}
host = host->next;
}
rrd_unlock();
return;
}
#define SQL_SELECT_HOST_MEMORY_MODE "SELECT memory_mode FROM chart WHERE host_id = @host_id LIMIT 1;"
@ -826,7 +823,7 @@ void aclk_update_retention(struct aclk_database_worker_config *wc)
return;
if (wc->host && rrdhost_flag_check(wc->host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
internal_error(true, "Skipping aclk_update_retention for host %s because context streaming is enabled", wc->host->hostname);
internal_error(true, "Skipping aclk_update_retention for host %s because context streaming is enabled", rrdhost_hostname(wc->host));
return;
}
@ -1093,9 +1090,9 @@ void queue_dimension_to_aclk(RRDDIM *rd, time_t last_updated)
memset(&dim_payload, 0, sizeof(dim_payload));
dim_payload.node_id = wc->node_id;
dim_payload.claim_id = claim_id;
dim_payload.name = rd->name;
dim_payload.id = rd->id;
dim_payload.chart_id = rd->rrdset->id;
dim_payload.name = rrddim_name(rd);
dim_payload.id = rrddim_id(rd);
dim_payload.chart_id = rrdset_id(rd->rrdset);
dim_payload.created_at.tv_sec = created_at;
dim_payload.last_timestamp.tv_sec = last_updated;
@ -1144,9 +1141,9 @@ void aclk_send_dimension_update(RRDDIM *rd)
rd->rrdset->rrdhost->dbsync_worker,
claim_id,
&rd->metric_uuid,
rd->id,
rd->name,
rd->rrdset->id,
rrddim_id(rd),
rrddim_name(rd),
rrdset_id(rd->rrdset),
first_entry_t,
live ? 0 : last_entry_t,
NULL);
@ -1155,9 +1152,9 @@ void aclk_send_dimension_update(RRDDIM *rd)
debug(
D_ACLK_SYNC,
"%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld)",
rd->rrdset->rrdhost->hostname,
rd->rrdset->name,
rd->name,
rrdhost_hostname(rd->rrdset->rrdhost),
rrdset_name(rd->rrdset),
rrddim_name(rd),
live,
first_entry_t,
last_entry_t);
@ -1165,9 +1162,9 @@ void aclk_send_dimension_update(RRDDIM *rd)
debug(
D_ACLK_SYNC,
"%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld) collected %ld seconds ago",
rd->rrdset->rrdhost->hostname,
rd->rrdset->name,
rd->name,
rrdhost_hostname(rd->rrdset->rrdhost),
rrdset_name(rd->rrdset),
rrddim_name(rd),
live,
first_entry_t,
last_entry_t,
@ -1280,15 +1277,15 @@ void sql_check_chart_liveness(RRDSET *st) {
if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
if (likely(st->dimensions && st->counter_done && !queue_chart_to_aclk(st))) {
debug(D_ACLK_SYNC,"Check chart liveness [%s] submit chart definition", st->name);
debug(D_ACLK_SYNC,"Check chart liveness [%s] submit chart definition", rrdset_name(st));
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
}
}
else
debug(D_ACLK_SYNC,"Check chart liveness [%s] chart definition already submitted", st->name);
debug(D_ACLK_SYNC,"Check chart liveness [%s] chart definition already submitted", rrdset_name(st));
time_t mark = now_realtime_sec();
debug(D_ACLK_SYNC,"Check chart liveness [%s] scanning dimensions", st->name);
debug(D_ACLK_SYNC,"Check chart liveness [%s] scanning dimensions", rrdset_name(st));
rrddim_foreach_read(rd, st) {
if (!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN))
queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, mark));

16
database/sqlite/sqlite_aclk_node.c
View File

@ -16,8 +16,8 @@ DICTIONARY *collectors_from_charts(RRDHOST *host, DICTIONARY *dict) {
rrdset_foreach_read(st, host) {
if (rrdset_is_available_for_viewers(st)) {
struct collector_info col = {
.plugin = st->plugin_name ? st->plugin_name : "",
.module = st->module_name ? st->module_name : ""
.plugin = rrdset_plugin_name(st),
.module = rrdset_module_name(st)
};
snprintfz(name, 499, "%s:%s", col.plugin, col.module);
dictionary_set(dict, name, &col, sizeof(struct collector_info));
@ -47,7 +47,7 @@ void sql_build_node_collectors(struct aclk_database_worker_config *wc)
dictionary_destroy(dict);
freez(upd_node_collectors.claim_id);
log_access("ACLK RES [%s (%s)]: NODE COLLECTORS SENT", wc->node_id, wc->host->hostname);
log_access("ACLK RES [%s (%s)]: NODE COLLECTORS SENT", wc->node_id, rrdhost_hostname(wc->host));
#else
UNUSED(wc);
#endif
@ -94,8 +94,8 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
netdata_mutex_unlock(&host->receiver_lock);
}
node_info.data.name = host->hostname;
node_info.data.os = (char *) host->os;
node_info.data.name = rrdhost_hostname(host);
node_info.data.os = rrdhost_os(host);
node_info.data.os_name = host->system_info->host_os_name;
node_info.data.os_version = host->system_info->host_os_version;
node_info.data.kernel_name = host->system_info->kernel_name;
@ -106,8 +106,8 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
node_info.data.memory = host->system_info->host_ram_total ? host->system_info->host_ram_total : "0";
node_info.data.disk_space = host->system_info->host_disk_space ? host->system_info->host_disk_space : "0";
node_info.data.version = host_version ? host_version : VERSION;
node_info.data.release_channel = (char *) get_release_channel();
node_info.data.timezone = (char *) host->abbrev_timezone;
node_info.data.release_channel = get_release_channel();
node_info.data.timezone = rrdhost_abbrev_timezone(host);
node_info.data.virtualization_type = host->system_info->virtualization ? host->system_info->virtualization : "unknown";
node_info.data.container_type = host->system_info->container ? host->system_info->container : "unknown";
node_info.data.custom_info = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", "");
@ -126,7 +126,7 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
node_info.data.host_labels_ptr = host->host_labels;
aclk_update_node_info(&node_info);
log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, wc->host->hostname, wc->host_guid, wc->host == localhost ? "parent" : "child");
log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, rrdhost_hostname(wc->host), wc->host_guid, wc->host == localhost ? "parent" : "child");
rrd_unlock();
freez(node_info.claim_id);

1
database/sqlite/sqlite_context.c
View File

@ -21,7 +21,6 @@ const char *database_context_cleanup[] = {
};
sqlite3 *db_context_meta = NULL;
/*
* Initialize the SQLite database
* Return 0 on success

169
database/sqlite/sqlite_functions.c
View File

@ -29,6 +29,7 @@ const char *database_config[] = {
"CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int);",
"CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);",
"CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);",
"CREATE INDEX IF NOT EXISTS ind_c2 on chart (host_id, context);",
"CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, "
"label_value text, date_created int, PRIMARY KEY (chart_id, label_key));",
"CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created);",
@ -408,6 +409,24 @@ int init_database_batch(sqlite3 *database, int rebuild, int init_type, const cha
return 0;
}
static void _uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv)
{
uuid_t uuid;
if ( argc != 1 ){
sqlite3_result_null(context);
return ;
}
int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid);
if (rc == -1) {
sqlite3_result_null(context);
return ;
}
sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);
}
/*
* Initialize the SQLite database
* Return 0 on success
@ -527,6 +546,9 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
for (int i = 0; i < MAX_PREPARED_STATEMENTS; i++)
(void)pthread_key_create(&key_pool[i], release_statement);
rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, _uuid_parse, 0, 0);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to register internal u2h function");
return 0;
}
@ -605,11 +627,11 @@ int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = sqlite3_bind_text(res, 2, rd->id, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 2, rrddim_id(rd), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = sqlite3_bind_text(res, 3, rd->name, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 3, rrddim_name(rd), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
@ -620,7 +642,7 @@ int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid)
}
else {
uuid_generate(*store_uuid);
status = sql_store_dimension(store_uuid, st->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor, rd->algorithm);
status = sql_store_dimension(store_uuid, st->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm);
if (unlikely(status))
error_report("Failed to store dimension metadata in the database");
}
@ -745,8 +767,11 @@ int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const
return 0;
rc = sql_store_chart(
chart_uuid, &st->rrdhost->host_uuid, st->type, id, name, st->family, st->context, st->title, st->units, st->plugin_name,
st->module_name, st->priority, st->update_every, st->chart_type, st->rrd_memory_mode, st->entries);
chart_uuid, &st->rrdhost->host_uuid, rrdset_type(st), id, name,
rrdset_family(st), rrdset_context(st), rrdset_title(st), rrdset_units(st),
rrdset_plugin_name(st), rrdset_module_name(st),
st->priority, st->update_every, st->chart_type,
st->rrd_memory_mode, st->entries);
return rc;
}
@ -762,7 +787,7 @@ uuid_t *create_chart_uuid(RRDSET *st, const char *id, const char *name)
#ifdef NETDATA_INTERNAL_CHECKS
char uuid_str[GUID_LEN + 1];
uuid_unparse_lower(*uuid, uuid_str);
debug(D_METADATALOG,"Generating uuid [%s] for chart %s under host %s", uuid_str, st->id, st->rrdhost->hostname);
debug(D_METADATALOG,"Generating uuid [%s] for chart %s under host %s", uuid_str, rrdset_id(st), rrdhost_hostname(st->rrdhost));
#endif
rc = update_chart_metadata(uuid, st, id, name);
@ -922,11 +947,11 @@ int sql_store_host_info(RRDHOST *host)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = bind_text_null(res, 2, host->hostname, 0);
rc = bind_text_null(res, 2, rrdhost_hostname(host), 0);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = bind_text_null(res, 3, host->registry_hostname, 1);
rc = bind_text_null(res, 3, rrdhost_registry_hostname(host), 1);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
@ -934,15 +959,15 @@ int sql_store_host_info(RRDHOST *host)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = bind_text_null(res, 5, host->os, 1);
rc = bind_text_null(res, 5, rrdhost_os(host), 1);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = bind_text_null(res, 6, host->timezone, 1);
rc = bind_text_null(res, 6, rrdhost_timezone(host), 1);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = bind_text_null(res, 7, host->tags, 1);
rc = bind_text_null(res, 7, rrdhost_tags(host), 1);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
@ -954,7 +979,7 @@ int sql_store_host_info(RRDHOST *host)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = bind_text_null(res, 10, host->abbrev_timezone, 1);
rc = bind_text_null(res, 10, rrdhost_abbrev_timezone(host), 1);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
@ -962,11 +987,11 @@ int sql_store_host_info(RRDHOST *host)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = bind_text_null(res, 12, host->program_name, 1);
rc = bind_text_null(res, 12, rrdhost_program_name(host), 1);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = bind_text_null(res, 13, host->program_version, 1);
rc = bind_text_null(res, 13, rrdhost_program_version(host), 1);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
@ -980,18 +1005,18 @@ int sql_store_host_info(RRDHOST *host)
int store_rc = sqlite3_step_monitored(res);
if (unlikely(store_rc != SQLITE_DONE))
error_report("Failed to store host %s, rc = %d", host->hostname, rc);
error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), rc);
rc = sqlite3_reset(res);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to reset statement to store host %s, rc = %d", host->hostname, rc);
error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
return !(store_rc == SQLITE_DONE);
bind_fail:
error_report("Failed to bind parameter to store host %s, rc = %d", host->hostname, rc);
error_report("Failed to bind parameter to store host %s, rc = %d", rrdhost_hostname(host), rc);
rc = sqlite3_reset(res);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to reset statement to store host %s, rc = %d", host->hostname, rc);
error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
return 1;
}
@ -1315,11 +1340,11 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb)
",\n\t\"memory_mode\": \"%s\""
",\n\t\"custom_info\": \"%s\""
",\n\t\"charts\": {"
, host->hostname
, host->program_version
, rrdhost_hostname(host)
, rrdhost_program_version(host)
, get_release_channel()
, host->os
, host->timezone
, rrdhost_os(host)
, rrdhost_timezone(host)
, host->rrd_update_every
, host->rrd_history_entries
, rrd_memory_mode_name(host->rrd_memory_mode)
@ -1410,7 +1435,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb)
"\n\t\t\t\"hostname\": \"%s\""
"\n\t\t}"
, (found > 0) ? "," : ""
, h->hostname
, rrdhost_hostname(h)
);
found++;
@ -1424,7 +1449,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb)
, "\n\t\t{"
"\n\t\t\t\"hostname\": \"%s\""
"\n\t\t}"
, host->hostname
, rrdhost_hostname(host)
);
}
@ -1445,13 +1470,13 @@ failed:
void free_temporary_host(RRDHOST *host)
{
if (host) {
freez(host->hostname);
freez((char *)host->os);
freez((char *)host->tags);
freez((char *)host->timezone);
freez(host->program_name);
freez(host->program_version);
freez(host->registry_hostname);
string_freez(host->hostname);
string_freez(host->os);
string_freez(host->tags);
string_freez(host->timezone);
string_freez(host->program_name);
string_freez(host->program_version);
string_freez(host->registry_hostname);
freez(host->system_info);
freez(host);
}
@ -1506,10 +1531,13 @@ RRDHOST *sql_create_host_by_uuid(char *hostname)
host = callocz(1, sizeof(RRDHOST));
set_host_properties(host, sqlite3_column_int(res, 2), RRD_MEMORY_MODE_DBENGINE, hostname,
(char *) sqlite3_column_text(res, 1), (const char *) uuid_str,
(char *) sqlite3_column_text(res, 3), (char *) sqlite3_column_text(res, 5),
(char *) sqlite3_column_text(res, 4), NULL, 0, NULL, NULL);
set_host_properties(host, sqlite3_column_int(res, 2),
RRD_MEMORY_MODE_DBENGINE,
(char *) sqlite3_column_text(res, 1),
(char *) sqlite3_column_text(res, 3),
(char *) sqlite3_column_text(res, 5),
(char *) sqlite3_column_text(res, 4),
NULL, 0, NULL, NULL);
uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
@ -1778,8 +1806,8 @@ static RRDDIM *create_rrdim_entry(ONEWAYALLOC *owa, RRDSET *st, char *id, char *
rrddim_flag_set(rd, RRDDIM_FLAG_NONE);
uuid_copy(rd->metric_uuid, *metric_uuid);
rd->id = onewayalloc_strdupz(owa, id);
rd->name = onewayalloc_strdupz(owa, name);
rd->id = string_strdupz(id);
rd->name = string_strdupz(name);
for(int tier = 0; tier < storage_tiers ;tier++) {
rd->tiers[tier] = onewayalloc_callocz(owa, 1, sizeof(*rd->tiers[tier]));
@ -1869,8 +1897,9 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para
if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) {
if (unlikely(st && !st->counter)) {
onewayalloc_freez(owa, st->context);
onewayalloc_freez(owa, (char *) st->name);
string_freez(st->context);
string_freez(st->name);
string_freez(st->id);
onewayalloc_freez(owa, st);
}
st = onewayalloc_callocz(owa, 1, sizeof(*st));
@ -1879,19 +1908,21 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para
snprintfz(
n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4),
(char *)sqlite3_column_text(res, 3));
st->name = onewayalloc_strdupz(owa, n);
st->name = string_strdupz(n);
st->update_every = sqlite3_column_int(res, 6);
st->counter = 0;
if (chart) {
st->context = onewayalloc_strdupz(owa, (char *)sqlite3_column_text(res, 8));
strncpyz(st->id, chart, RRD_ID_LENGTH_MAX);
}
st->context = string_strdupz((char *)sqlite3_column_text(res, 8));
st->id = string_strdupz(chart);
} else
st->id = string_strdupz(n);
uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7));
st->last_entry_t = 0;
st->rrdhost = host;
}
if (unlikely(find_dimension_first_last_t(machine_guid, (char *)st->name, (char *)sqlite3_column_text(res, 1),
if (unlikely(find_dimension_first_last_t(machine_guid, (char *)rrdset_name(st), (char *)sqlite3_column_text(res, 1),
(uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t,
&rrdeng_uuid, 0)))
continue;
@ -1909,13 +1940,14 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para
}
if (st) {
if (!st->counter) {
onewayalloc_freez(owa,st->context);
onewayalloc_freez(owa,(char *)st->name);
string_freez(st->context);
string_freez(st->name);
string_freez(st->id);
onewayalloc_freez(owa,st);
}
else
if (!st->context && context)
st->context = onewayalloc_strdupz(owa,context);
st->context = string_strdupz(context);
}
failed:
@ -2063,7 +2095,7 @@ void compute_chart_hash(RRDSET *st)
char priority_str[32];
if (rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
internal_error(true, "Skipping compute_chart_hash for host %s because context streaming is enabled", st->rrdhost->hostname);
internal_error(true, "Skipping compute_chart_hash for host %s because context streaming is enabled", rrdhost_hostname(st->rrdhost));
return;
}
@ -2072,15 +2104,14 @@ void compute_chart_hash(RRDSET *st)
evpctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
//EVP_DigestUpdate(evpctx, st->type, strlen(st->type));
EVP_DigestUpdate(evpctx, st->id, strlen(st->id));
EVP_DigestUpdate(evpctx, st->name, strlen(st->name));
EVP_DigestUpdate(evpctx, st->family, strlen(st->family));
EVP_DigestUpdate(evpctx, st->context, strlen(st->context));
EVP_DigestUpdate(evpctx, st->title, strlen(st->title));
EVP_DigestUpdate(evpctx, st->units, strlen(st->units));
EVP_DigestUpdate(evpctx, st->plugin_name, strlen(st->plugin_name));
if (st->module_name)
EVP_DigestUpdate(evpctx, st->module_name, strlen(st->module_name));
EVP_DigestUpdate(evpctx, rrdset_id(st), string_strlen(st->id));
EVP_DigestUpdate(evpctx, rrdset_name(st), string_strlen(st->name));
EVP_DigestUpdate(evpctx, rrdset_family(st), string_strlen(st->family));
EVP_DigestUpdate(evpctx, rrdset_context(st), string_strlen(st->context));
EVP_DigestUpdate(evpctx, rrdset_title(st), string_strlen(st->title));
EVP_DigestUpdate(evpctx, rrdset_units(st), string_strlen(st->units));
EVP_DigestUpdate(evpctx, rrdset_plugin_name(st), string_strlen(st->plugin_name));
EVP_DigestUpdate(evpctx, rrdset_module_name(st), string_strlen(st->module_name));
// EVP_DigestUpdate(evpctx, priority_str, strlen(priority_str));
EVP_DigestUpdate(evpctx, &st->priority, sizeof(st->priority));
EVP_DigestUpdate(evpctx, &st->chart_type, sizeof(st->chart_type));
@ -2096,15 +2127,15 @@ void compute_chart_hash(RRDSET *st)
(void)sql_store_chart_hash(
(uuid_t *)&hash_value,
st->chart_uuid,
st->type,
st->id,
st->name,
st->family,
st->context,
st->title,
st->units,
st->plugin_name,
st->module_name,
rrdset_type(st),
rrdset_id(st),
rrdset_name(st),
rrdset_family(st),
rrdset_context(st),
rrdset_title(st),
rrdset_units(st),
rrdset_plugin_name(st),
rrdset_module_name(st),
st->priority,
st->chart_type);
#else
@ -2224,7 +2255,7 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id)
char host_guid[GUID_LEN + 1];
uuid_unparse_lower(*host_id, host_guid);
rrd_wrlock();
host = rrdhost_find_by_guid(host_guid, 0);
host = rrdhost_find_by_guid(host_guid);
if (likely(host))
set_host_node_id(host, node_id);
rrd_unlock();
@ -2439,7 +2470,7 @@ struct node_instance_list *get_node_list(void)
uuid_copy(node_list[row].host_id, *host_id);
node_list[row].queryable = 1;
uuid_unparse_lower(*host_id, host_guid);
RRDHOST *host = rrdhost_find_by_guid(host_guid, 0);
RRDHOST *host = rrdhost_find_by_guid(host_guid);
node_list[row].live = host && (host == localhost || host->receiver) ? 1 : 0;
node_list[row].hops = (host && host->system_info) ? host->system_info->hops :
uuid_compare(*host_id, localhost->host_uuid) ? 1 : 0;
@ -2677,7 +2708,7 @@ void sql_store_host_labels(RRDHOST *host)
{
int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid);
if (rc != SQLITE_OK)
error_report("Failed to remove old host labels for host %s", host->hostname);
error_report("Failed to remove old host labels for host %s", rrdhost_hostname(host));
rrdlabels_walkthrough_read(host->host_labels, save_host_label_callback, host);
}

195
database/sqlite/sqlite_health.c
View File

@ -15,7 +15,7 @@ int sql_create_health_log_table(RRDHOST *host) {
if (unlikely(!db_meta)) {
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
error_report("HEALTH [%s]: Database has not been initialized", host->hostname);
error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
return 1;
}
@ -26,7 +26,7 @@ int sql_create_health_log_table(RRDHOST *host) {
rc = sqlite3_exec_monitored(db_meta, command, 0, 0, &err_msg);
if (rc != SQLITE_OK) {
error_report("HEALTH [%s]: SQLite error during creation of health log table, rc = %d (%s)", host->hostname, rc, err_msg);
error_report("HEALTH [%s]: SQLite error during creation of health log table, rc = %d (%s)", rrdhost_hostname(host), rc, err_msg);
sqlite3_free(err_msg);
return 1;
}
@ -49,7 +49,7 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
if (unlikely(!db_meta)) {
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
error_report("HEALTH [%s]: Database has not been initialized", host->hostname);
error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
return;
}
@ -60,7 +60,7 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
if (unlikely(rc != SQLITE_OK)) {
error_report("HEALTH [%s]: Failed to prepare statement for SQL_UPDATE_HEALTH_LOG", host->hostname);
error_report("HEALTH [%s]: Failed to prepare statement for SQL_UPDATE_HEALTH_LOG", rrdhost_hostname(host));
return;
}
@ -96,12 +96,12 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
rc = execute_insert(res);
if (unlikely(rc != SQLITE_DONE)) {
error_report("HEALTH [%s]: Failed to update health log, rc = %d", host->hostname, rc);
error_report("HEALTH [%s]: Failed to update health log, rc = %d", rrdhost_hostname(host), rc);
}
failed:
if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
error_report("HEALTH [%s]: Failed to finalize the prepared statement for updating health log.", host->hostname);
error_report("HEALTH [%s]: Failed to finalize the prepared statement for updating health log.", rrdhost_hostname(host));
return;
}
@ -122,7 +122,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
if (unlikely(!db_meta)) {
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
error_report("HEALTH [%s]: Database has not been initialized", host->hostname);
error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
return;
}
@ -133,11 +133,11 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
if (unlikely(rc != SQLITE_OK)) {
error_report("HEALTH [%s]: Failed to prepare statement for SQL_INSERT_HEALTH_LOG", host->hostname);
error_report("HEALTH [%s]: Failed to prepare statement for SQL_INSERT_HEALTH_LOG", rrdhost_hostname(host));
return;
}
rc = sqlite3_bind_text(res, 1, host->hostname, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 1, rrdhost_hostname(host), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind hostname parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
@ -215,49 +215,49 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
goto failed;
}
rc = sqlite3_bind_text(res, 14, ae->name, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 14, ae_name(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind name parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 15, ae->chart, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 15, ae_chart_name(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind chart parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 16, ae->family, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 16, ae_family(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind family parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 17, ae->exec, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 17, ae_exec(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind exec parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 18, ae->recipient, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 18, ae_recipient(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind recipient parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 19, ae->source, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 19, ae_source(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind source parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 20, ae->units, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 20, ae_units(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind host_id parameter to store node instance information");
goto failed;
}
rc = sqlite3_bind_text(res, 21, ae->info, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 21, ae_info(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind info parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
@ -305,25 +305,25 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
goto failed;
}
rc = sqlite3_bind_text(res, 29, ae->classification, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 29, ae_classification(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind classification parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 30, ae->component, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 30, ae_component(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind component parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 31, ae->type, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 31, ae_type(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind type parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
}
rc = sqlite3_bind_text(res, 32, ae->chart_context, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 32, ae_chart_context(ae), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind chart_context parameter for SQL_INSERT_HEALTH_LOG");
goto failed;
@ -331,7 +331,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
rc = execute_insert(res);
if (unlikely(rc != SQLITE_DONE)) {
error_report("HEALTH [%s]: Failed to execute SQL_INSERT_HEALTH_LOG, rc = %d", host->hostname, rc);
error_report("HEALTH [%s]: Failed to execute SQL_INSERT_HEALTH_LOG, rc = %d", rrdhost_hostname(host), rc);
goto failed;
}
@ -340,7 +340,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
failed:
if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
error_report("HEALTH [%s]: Failed to finalize the prepared statement for inserting to health log.", host->hostname);
error_report("HEALTH [%s]: Failed to finalize the prepared statement for inserting to health log.", rrdhost_hostname(host));
return;
}
@ -436,7 +436,7 @@ void sql_health_alarm_log_count(RRDHOST *host) {
if (unlikely(rc != SQLITE_OK))
error_report("Failed to finalize the prepared statement to count health log entries from db");
info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", host->hostname, uuid_str, host->health_log_entries_written);
info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", rrdhost_hostname(host), uuid_str, host->health_log_entries_written);
}
#define SQL_INJECT_REMOVED(guid, guid2) "insert into health_log_%s (hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, " \
@ -607,7 +607,7 @@ void sql_check_removed_alerts_state(char *uuid_str)
#define SQL_LOAD_HEALTH_LOG(guid,limit) "SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type, chart_context FROM (SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type, chart_context FROM health_log_%s order by unique_id desc limit %u) order by unique_id asc;", guid, limit
void sql_health_alarm_log_load(RRDHOST *host) {
sqlite3_stmt *res = NULL;
int rc;
int ret;
ssize_t errored = 0, loaded = 0;
char command[MAX_HEALTH_SQL_SIZE + 1];
@ -615,7 +615,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
if (unlikely(!db_meta)) {
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
error_report("HEALTH [%s]: Database has not been initialized", host->hostname);
error_report("HEALTH [%s]: Database has not been initialized", rrdhost_hostname(host));
return;
}
@ -626,47 +626,52 @@ void sql_health_alarm_log_load(RRDHOST *host) {
snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_LOAD_HEALTH_LOG(uuid_str, host->health_log.max));
rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
if (unlikely(rc != SQLITE_OK)) {
error_report("HEALTH [%s]: Failed to prepare sql statement to load health log.", host->hostname);
ret = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
if (unlikely(ret != SQLITE_OK)) {
error_report("HEALTH [%s]: Failed to prepare sql statement to load health log.", rrdhost_hostname(host));
return;
}
netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
DICTIONARY *all_rrdcalcs = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
RRDCALC *rc;
foreach_rrdcalc_in_rrdhost(host, rc)
dictionary_set(all_rrdcalcs, rrdcalc_name(rc), rc, sizeof(*rc));
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
ALARM_ENTRY *ae = NULL;
// check that we have valid ids
uint32_t unique_id = (uint32_t) sqlite3_column_int64(res, 1);
if(!unique_id) {
error_report("HEALTH [%s]: Got invalid unique id. Ignoring it.", host->hostname);
error_report("HEALTH [%s]: Got invalid unique id. Ignoring it.", rrdhost_hostname(host));
errored++;
continue;
}
uint32_t alarm_id = (uint32_t) sqlite3_column_int64(res, 2);
if(!alarm_id) {
error_report("HEALTH [%s]: Got invalid alarm id. Ignoring it.", host->hostname);
error_report("HEALTH [%s]: Got invalid alarm id. Ignoring it.", rrdhost_hostname(host));
errored++;
continue;
}
//need name, chart and family
if (sqlite3_column_type(res, 13) == SQLITE_NULL) {
error_report("HEALTH [%s]: Got null name field. Ignoring it.", host->hostname);
error_report("HEALTH [%s]: Got null name field. Ignoring it.", rrdhost_hostname(host));
errored++;
continue;
}
if (sqlite3_column_type(res, 14) == SQLITE_NULL) {
error_report("HEALTH [%s]: Got null chart field. Ignoring it.", host->hostname);
error_report("HEALTH [%s]: Got null chart field. Ignoring it.", rrdhost_hostname(host));
errored++;
continue;
}
if (sqlite3_column_type(res, 15) == SQLITE_NULL) {
error_report("HEALTH [%s]: Got null family field. Ignoring it.", host->hostname);
error_report("HEALTH [%s]: Got null family field. Ignoring it.", rrdhost_hostname(host));
errored++;
continue;
}
@ -675,18 +680,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
time_t last_repeat = 0;
last_repeat = (time_t)sqlite3_column_int64(res, 27);
RRDCALC *rc = alarm_max_last_repeat(host, (char *) sqlite3_column_text(res, 14), simple_hash((char *) sqlite3_column_text(res, 14)));
if (!rc) {
for(rc = host->alarms; rc ; rc = rc->next) {
RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_name, (avl_t *)rc);
if(rdcmp != rc) {
error("Cannot insert the alarm index ID using log %s", rc->name);
}
}
rc = alarm_max_last_repeat(host, (char *) sqlite3_column_text(res, 14), simple_hash((char *) sqlite3_column_text(res, 14)));
}
rc = dictionary_get(all_rrdcalcs, (char *) sqlite3_column_text(res, 14));
if(unlikely(rc)) {
if (rrdcalc_isrepeating(rc)) {
rc->last_repeat = last_repeat;
@ -719,36 +713,32 @@ void sql_health_alarm_log_load(RRDHOST *host) {
ae->exec_run_timestamp = (time_t) sqlite3_column_int64(res, 11);
ae->delay_up_to_timestamp = (time_t) sqlite3_column_int64(res, 12);
ae->name = strdupz((char *) sqlite3_column_text(res, 13));
ae->hash_name = simple_hash(ae->name);
ae->chart = strdupz((char *) sqlite3_column_text(res, 14));
ae->hash_chart = simple_hash(ae->chart);
ae->family = strdupz((char *) sqlite3_column_text(res, 15));
ae->name = string_strdupz((char *) sqlite3_column_text(res, 13));
ae->chart = string_strdupz((char *) sqlite3_column_text(res, 14));
ae->family = string_strdupz((char *) sqlite3_column_text(res, 15));
if (sqlite3_column_type(res, 16) != SQLITE_NULL)
ae->exec = strdupz((char *) sqlite3_column_text(res, 16));
ae->exec = string_strdupz((char *) sqlite3_column_text(res, 16));
else
ae->exec = NULL;
if (sqlite3_column_type(res, 17) != SQLITE_NULL)
ae->recipient = strdupz((char *) sqlite3_column_text(res, 17));
ae->recipient = string_strdupz((char *) sqlite3_column_text(res, 17));
else
ae->recipient = NULL;
if (sqlite3_column_type(res, 18) != SQLITE_NULL)
ae->source = strdupz((char *) sqlite3_column_text(res, 18));
ae->source = string_strdupz((char *) sqlite3_column_text(res, 18));
else
ae->source = NULL;
if (sqlite3_column_type(res, 19) != SQLITE_NULL)
ae->units = strdupz((char *) sqlite3_column_text(res, 19));
ae->units = string_strdupz((char *) sqlite3_column_text(res, 19));
else
ae->units = NULL;
if (sqlite3_column_type(res, 20) != SQLITE_NULL)
ae->info = strdupz((char *) sqlite3_column_text(res, 20));
ae->info = string_strdupz((char *) sqlite3_column_text(res, 20));
else
ae->info = NULL;
@ -763,30 +753,30 @@ void sql_health_alarm_log_load(RRDHOST *host) {
ae->last_repeat = last_repeat;
if (sqlite3_column_type(res, 28) != SQLITE_NULL)
ae->classification = strdupz((char *) sqlite3_column_text(res, 28));
ae->classification = string_strdupz((char *) sqlite3_column_text(res, 28));
else
ae->classification = NULL;
if (sqlite3_column_type(res, 29) != SQLITE_NULL)
ae->component = strdupz((char *) sqlite3_column_text(res, 29));
ae->component = string_strdupz((char *) sqlite3_column_text(res, 29));
else
ae->component = NULL;
if (sqlite3_column_type(res, 30) != SQLITE_NULL)
ae->type = strdupz((char *) sqlite3_column_text(res, 30));
ae->type = string_strdupz((char *) sqlite3_column_text(res, 30));
else
ae->type = NULL;
if (sqlite3_column_type(res, 31) != SQLITE_NULL)
ae->chart_context = strdupz((char *) sqlite3_column_text(res, 31));
ae->chart_context = string_strdupz((char *) sqlite3_column_text(res, 31));
else
ae->chart_context = NULL;
char value_string[100 + 1];
freez(ae->old_value_string);
freez(ae->new_value_string);
ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
string_freez(ae->old_value_string);
string_freez(ae->new_value_string);
ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1));
ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1));
ae->next = host->health_log.alarms;
host->health_log.alarms = ae;
@ -800,6 +790,9 @@ void sql_health_alarm_log_load(RRDHOST *host) {
loaded++;
}
dictionary_destroy(all_rrdcalcs);
all_rrdcalcs = NULL;
netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);
if(!host->health_max_unique_id) host->health_max_unique_id = (uint32_t)now_realtime_sec();
@ -809,10 +802,10 @@ void sql_health_alarm_log_load(RRDHOST *host) {
if (unlikely(!host->health_log.next_alarm_id || host->health_log.next_alarm_id <= host->health_max_alarm_id))
host->health_log.next_alarm_id = host->health_max_alarm_id + 1;
info("HEALTH [%s]: Table health_log_%s, loaded %zd alarm entries, errors in %zd entries.", host->hostname, uuid_str, loaded, errored);
info("HEALTH [%s]: Table health_log_%s, loaded %zd alarm entries, errors in %zd entries.", rrdhost_hostname(host), uuid_str, loaded, errored);
rc = sqlite3_finalize(res);
if (unlikely(rc != SQLITE_OK))
ret = sqlite3_finalize(res);
if (unlikely(ret != SQLITE_OK))
error_report("Failed to finalize the health log read statement");
sql_health_alarm_log_count(host);
@ -854,154 +847,154 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
goto bind_fail;
param++;
if (cfg->alarm && *cfg->alarm)
rc = sqlite3_bind_text(res, 2, cfg->alarm, -1, SQLITE_STATIC);
if (cfg->alarm)
rc = sqlite3_bind_text(res, 2, string2str(cfg->alarm), -1, SQLITE_STATIC);
else
rc = sqlite3_bind_null(res, 2);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
if (cfg->template_key && *cfg->template_key)
rc = sqlite3_bind_text(res, 3, cfg->template_key, -1, SQLITE_STATIC);
if (cfg->template_key)
rc = sqlite3_bind_text(res, 3, string2str(cfg->template_key), -1, SQLITE_STATIC);
else
rc = sqlite3_bind_null(res, 3);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 4, cfg->on, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 4, string2str(cfg->on), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 5, cfg->classification, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 5, string2str(cfg->classification), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 6, cfg->component, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 6, string2str(cfg->component), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 7, cfg->type, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 7, string2str(cfg->type), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 8, cfg->os, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 8, string2str(cfg->os), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 9, cfg->host, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 9, string2str(cfg->host), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 10, cfg->lookup, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 10, string2str(cfg->lookup), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 11, cfg->every, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 11, string2str(cfg->every), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 12, cfg->units, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 12, string2str(cfg->units), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 13, cfg->calc, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 13, string2str(cfg->calc), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 14, cfg->families, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 14, string2str(cfg->families), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 15, cfg->plugin, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 15, string2str(cfg->plugin), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 16, cfg->module, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 16, string2str(cfg->module), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 17, cfg->charts, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 17, string2str(cfg->charts), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 18, cfg->green, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 18, string2str(cfg->green), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 19, cfg->red, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 19, string2str(cfg->red), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 20, cfg->warn, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 20, string2str(cfg->warn), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 21, cfg->crit, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 21, string2str(cfg->crit), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 22, cfg->exec, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 22, string2str(cfg->exec), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 23, cfg->to, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 23, string2str(cfg->to), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 24, cfg->info, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 24, string2str(cfg->info), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 25, cfg->delay, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 25, string2str(cfg->delay), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 26, cfg->options, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 26, string2str(cfg->options), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 27, cfg->repeat, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 27, string2str(cfg->repeat), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 28, cfg->host_labels, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 28, string2str(cfg->host_labels), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
if (cfg->p_db_lookup_after) {
param++;
rc = sqlite3_bind_text(res, 29, cfg->p_db_lookup_dimensions, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 29, string2str(cfg->p_db_lookup_dimensions), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 30, cfg->p_db_lookup_method, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(res, 30, string2str(cfg->p_db_lookup_method), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
@ -1071,7 +1064,7 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
skip hash calculations
*/
#if !defined DISABLE_CLOUD && defined ENABLE_HTTPS
#define DIGEST_ALERT_CONFIG_VAL(v) ((v) ? EVP_DigestUpdate(evpctx, (v), strlen((v))) : EVP_DigestUpdate(evpctx, "", 1))
#define DIGEST_ALERT_CONFIG_VAL(v) EVP_DigestUpdate(evpctx, (string2str(v)), string_strlen(v))
#endif
int alert_hash_and_store_config(
uuid_t hash_id,

10
exporting/check_filters.c
View File

@ -25,7 +25,7 @@ int rrdhost_is_exportable(struct instance *instance, RRDHOST *host)
RRDHOST_FLAGS *flags = &host->exporting_flags[instance->index];
if (unlikely((*flags & (RRDHOST_FLAG_EXPORTING_SEND | RRDHOST_FLAG_EXPORTING_DONT_SEND)) == 0)) {
char *host_name = (host == localhost) ? "localhost" : host->hostname;
const char *host_name = (host == localhost) ? "localhost" : rrdhost_hostname(host);
if (!instance->config.hosts_pattern || simple_pattern_matches(instance->config.hosts_pattern, host_name)) {
*flags |= RRDHOST_FLAG_EXPORTING_SEND;
@ -69,22 +69,22 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st)
if(unlikely(!(*flags & RRDSET_FLAG_EXPORTING_SEND))) {
// we have not checked this chart
if(simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name))
if(simple_pattern_matches(instance->config.charts_pattern, rrdset_id(st)) || simple_pattern_matches(instance->config.charts_pattern, rrdset_name(st)))
*flags |= RRDSET_FLAG_EXPORTING_SEND;
else {
*flags |= RRDSET_FLAG_EXPORTING_IGNORE;
debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", st->id, host->hostname);
debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", rrdset_id(st), rrdhost_hostname(host));
return 0;
}
}
if(unlikely(!rrdset_is_available_for_exporting_and_alarms(st))) {
debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", st->id, host->hostname);
debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", rrdset_id(st), rrdhost_hostname(host));
return 0;
}
if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode));
debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode));
return 0;
}

16
exporting/graphite/graphite.c
View File

@ -123,24 +123,24 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD
char chart_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
chart_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st),
RRD_ID_LENGTH_MAX);
char dimension_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
dimension_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
RRD_ID_LENGTH_MAX);
buffer_sprintf(
instance->buffer,
"%s.%s.%s.%s%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n",
instance->config.prefix,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
chart_name,
dimension_name,
(host->tags) ? ";" : "",
(host->tags) ? host->tags : "",
(host->tags) ? rrdhost_tags(host) : "",
(instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
rd->last_collected_value,
(unsigned long long)rd->last_collected_time.tv_sec);
@ -163,13 +163,13 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
char chart_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
chart_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st),
RRD_ID_LENGTH_MAX);
char dimension_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
dimension_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
RRD_ID_LENGTH_MAX);
time_t last_t;
@ -182,11 +182,11 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
instance->buffer,
"%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
instance->config.prefix,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
chart_name,
dimension_name,
(host->tags) ? ";" : "",
(host->tags) ? host->tags : "",
(host->tags) ? rrdhost_tags(host) : "",
(instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
value,
(unsigned long long)last_t);

42
exporting/json/json.c
View File

@ -145,7 +145,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
RRDSET *st = rd->rrdset;
RRDHOST *host = st->rrdhost;
const char *tags_pre = "", *tags_post = "", *tags = host->tags;
const char *tags_pre = "", *tags_post = "", *tags = rrdhost_tags(host);
if (!tags)
tags = "";
@ -187,21 +187,20 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
"\"timestamp\":%llu}",
instance->config.prefix,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
tags_pre,
tags,
tags_post,
instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
st->id,
st->name,
st->family,
st->context,
st->type,
st->units,
rd->id,
rd->name,
rrdset_id(st),
rrdset_name(st),
rrdset_family(st),
rrdset_context(st),
rrdset_type(st),
rrdset_units(st),
rrddim_id(rd),
rrddim_name(rd),
rd->last_collected_value,
(unsigned long long)rd->last_collected_time.tv_sec);
@ -231,7 +230,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
if(isnan(value))
return 0;
const char *tags_pre = "", *tags_post = "", *tags = host->tags;
const char *tags_pre = "", *tags_post = "", *tags = rrdhost_tags(host);
if (!tags)
tags = "";
@ -272,21 +271,20 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
"\"timestamp\": %llu}",
instance->config.prefix,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
tags_pre,
tags,
tags_post,
instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
st->id,
st->name,
st->family,
st->context,
st->type,
st->units,
rd->id,
rd->name,
rrdset_id(st),
rrdset_name(st),
rrdset_family(st),
rrdset_context(st),
rrdset_type(st),
rrdset_units(st),
rrddim_id(rd),
rrddim_name(rd),
value,
(unsigned long long)last_t);

32
exporting/opentsdb/opentsdb.c
View File

@ -177,13 +177,13 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
char chart_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
chart_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st),
RRD_ID_LENGTH_MAX);
char dimension_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
dimension_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
RRD_ID_LENGTH_MAX);
buffer_sprintf(
@ -194,9 +194,9 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
dimension_name,
(unsigned long long)rd->last_collected_time.tv_sec,
rd->last_collected_value,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
(host->tags) ? rrdhost_tags(host) : "",
(instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
return 0;
@ -217,13 +217,13 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
char chart_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
chart_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st),
RRD_ID_LENGTH_MAX);
char dimension_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
dimension_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
RRD_ID_LENGTH_MAX);
time_t last_t;
@ -240,9 +240,9 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
dimension_name,
(unsigned long long)last_t,
value,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
(host->tags) ? rrdhost_tags(host) : "",
(instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
return 0;
@ -309,13 +309,13 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
char chart_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
chart_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st),
RRD_ID_LENGTH_MAX);
char dimension_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
dimension_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
RRD_ID_LENGTH_MAX);
if (buffer_strlen((BUFFER *)instance->buffer) > 2)
@ -336,9 +336,9 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
dimension_name,
(unsigned long long)rd->last_collected_time.tv_sec,
rd->last_collected_value,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
(host->tags) ? rrdhost_tags(host) : "",
instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
return 0;
@ -359,13 +359,13 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
char chart_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
chart_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st),
RRD_ID_LENGTH_MAX);
char dimension_name[RRD_ID_LENGTH_MAX + 1];
exporting_name_copy(
dimension_name,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
RRD_ID_LENGTH_MAX);
time_t last_t;
@ -392,9 +392,9 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
dimension_name,
(unsigned long long)last_t,
value,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
(host->tags) ? rrdhost_tags(host) : "",
instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
return 0;

12
exporting/process_data.c
View File

@ -110,9 +110,9 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
debug(
D_EXPORTING,
"EXPORTING: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu",
host->hostname,
st->id,
rd->id,
rrdhost_hostname(host),
rrdset_id(st),
rrddim_id(rd),
(unsigned long)after,
(unsigned long)before,
(unsigned long)first_t,
@ -142,9 +142,9 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
debug(
D_EXPORTING,
"EXPORTING: %s.%s.%s: no values stored in database for range %lu to %lu",
host->hostname,
st->id,
rd->id,
rrdhost_hostname(host),
rrdset_id(st),
rrddim_id(rd),
(unsigned long)after,
(unsigned long)before);
return NAN;

94
exporting/prometheus/prometheus.c
View File

@ -9,9 +9,9 @@
static int is_matches_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN *filter) {
if (instance->config.options & EXPORTING_OPTION_SEND_NAMES) {
return simple_pattern_matches(filter, st->name);
return simple_pattern_matches(filter, rrdset_name(st));
}
return simple_pattern_matches(filter, st->id);
return simple_pattern_matches(filter, rrdset_id(st));
}
/**
@ -48,8 +48,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN
debug(
D_EXPORTING,
"EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.",
st->id,
host->hostname);
rrdset_id(st),
rrdhost_hostname(host));
return 0;
}
}
@ -58,8 +58,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN
debug(
D_EXPORTING,
"EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.",
st->id,
host->hostname);
rrdset_id(st),
rrdhost_hostname(host));
return 0;
}
@ -69,8 +69,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN
debug(
D_EXPORTING,
"EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting connector requires database access.",
st->id,
host->hostname,
rrdset_id(st),
rrdhost_hostname(host),
rrd_memory_mode_name(host->rrd_memory_mode));
return 0;
}
@ -349,8 +349,9 @@ struct host_variables_callback_options {
* @param data callback options.
* @return Returns 1 if the chart can be sent, 0 otherwise.
*/
static int print_host_variables(RRDVAR *rv, void *data)
{
static int print_host_variables(const char *name __maybe_unused, void *rv_ptr, void *data) {
RRDVAR *rv = rv_ptr;
struct host_variables_callback_options *opts = data;
if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) {
@ -366,7 +367,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
if (isnan(value) || isinf(value)) {
if (opts->output_options & PROMETHEUS_OUTPUT_HELP)
buffer_sprintf(
opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value)) ? "NAN" : "INF");
opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rrdvar_name(rv), (isnan(value)) ? "NAN" : "INF");
return 0;
}
@ -378,7 +379,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
label_post = "}";
}
prometheus_name_copy(opts->name, rv->name, sizeof(opts->name));
prometheus_name_copy(opts->name, rrdvar_name(rv), sizeof(opts->name));
if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
buffer_sprintf(
@ -445,17 +446,17 @@ static void generate_as_collected_prom_help(BUFFER *wb, struct gen_parameters *p
wb,
"%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * ",
p->suffix,
(p->output_options & PROMETHEUS_OUTPUT_NAMES && p->st->name) ? p->st->name : p->st->id,
p->st->context,
p->st->family,
(p->output_options & PROMETHEUS_OUTPUT_NAMES && p->rd->name) ? p->rd->name : p->rd->id);
(p->output_options & PROMETHEUS_OUTPUT_NAMES && p->st->name) ? rrdset_name(p->st) : rrdset_id(p->st),
rrdset_context(p->st),
rrdset_family(p->st),
(p->output_options & PROMETHEUS_OUTPUT_NAMES && p->rd->name) ? rrddim_name(p->rd) : rrddim_id(p->rd));
if (prometheus_collector)
buffer_sprintf(wb, "1 / 1");
else
buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT, p->rd->multiplier, p->rd->divisor);
buffer_sprintf(wb, " %s %s (%s)\n", p->relation, p->st->units, p->type);
buffer_sprintf(wb, " %s %s (%s)\n", p->relation, rrdset_units(p->st), p->type);
}
/**
@ -521,7 +522,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
rrdhost_rdlock(host);
char hostname[PROMETHEUS_ELEMENT_MAX + 1];
prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(hostname, rrdhost_hostname(host), PROMETHEUS_ELEMENT_MAX);
format_host_labels_prometheus(instance, host);
@ -529,8 +530,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
wb,
"netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"",
hostname,
host->program_name,
host->program_version);
rrdhost_program_name(host),
rrdhost_program_version(host));
if (instance->labels_buffer && *buffer_tostring(instance->labels_buffer)) {
buffer_sprintf(wb, ",%s", buffer_tostring(instance->labels_buffer));
@ -551,15 +552,19 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
// send custom variables set for the host
if (output_options & PROMETHEUS_OUTPUT_VARIABLES) {
struct host_variables_callback_options opts = { .host = host,
.wb = wb,
.labels = (labels[0] == ',') ? &labels[1] : labels,
.exporting_options = exporting_options,
.output_options = output_options,
.prefix = prefix,
.now = now_realtime_sec(),
.host_header_printed = 0 };
foreach_host_variable_callback(host, print_host_variables, &opts);
struct host_variables_callback_options opts = {
.host = host,
.wb = wb,
.labels = (labels[0] == ',') ? &labels[1] : labels,
.exporting_options = exporting_options,
.output_options = output_options,
.prefix = prefix,
.now = now_realtime_sec(),
.host_header_printed = 0
};
rrdvar_walkthrough_read(host->rrdvar_root_index, print_host_variables, &opts);
}
// for each chart
@ -575,10 +580,9 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
char family[PROMETHEUS_ELEMENT_MAX + 1];
char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
prometheus_label_copy(
chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(family, rrdset_family(st), PROMETHEUS_ELEMENT_MAX);
prometheus_name_copy(context, rrdset_context(st), PROMETHEUS_ELEMENT_MAX);
int as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AS_COLLECTED);
int homogeneous = 1;
@ -590,13 +594,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
if (rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS))
homogeneous = 0;
if (st->module_name && !strcmp(st->module_name, "prometheus"))
if (!strcmp(rrdset_module_name(st), "prometheus"))
prometheus_collector = 1;
} else {
if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE &&
!(output_options & PROMETHEUS_OUTPUT_HIDEUNITS))
prometheus_units_copy(
units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS);
units, rrdset_units(st), PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS);
}
if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
@ -604,10 +608,10 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
wb,
"\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n",
(homogeneous) ? "homogeneous" : "heterogeneous",
(output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id,
st->context,
st->family,
st->units);
(output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st),
rrdset_context(st),
rrdset_family(st),
rrdset_units(st));
// for each dimension
RRDDIM *rd;
@ -651,7 +655,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
prometheus_label_copy(
dimension,
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
PROMETHEUS_ELEMENT_MAX);
if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
@ -667,7 +671,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
prometheus_name_copy(
dimension,
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
PROMETHEUS_ELEMENT_MAX);
if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
@ -694,7 +698,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
prometheus_label_copy(
dimension,
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
PROMETHEUS_ELEMENT_MAX);
if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
@ -705,8 +709,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
context,
units,
suffix,
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
st->units,
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
rrdset_units(st),
(unsigned long long)first_time,
(unsigned long long)last_time);
@ -809,7 +813,7 @@ static inline time_t prometheus_preparation(
buffer_sprintf(
wb,
"# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s, time range %lu to %lu\n\n",
host->hostname,
rrdhost_hostname(host),
(first_seen) ? "FIRST SEEN " : "",
server,
mode,

38
exporting/prometheus/remote_write/remote_write.c
View File

@ -171,12 +171,12 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host
char hostname[PROMETHEUS_ELEMENT_MAX + 1];
prometheus_label_copy(
hostname,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
PROMETHEUS_ELEMENT_MAX);
add_host_info(
connector_specific_data->write_request,
"netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
"netdata_info", hostname, rrdhost_program_name(host), rrdhost_program_version(host), now_realtime_usec() / USEC_PER_MS);
if (unlikely(sending_labels_configured(instance))) {
struct format_remote_write_label_callback tmp = {
@ -200,10 +200,10 @@ int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st)
{
prometheus_label_copy(
chart,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st),
PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(family, rrdset_family(st), PROMETHEUS_ELEMENT_MAX);
prometheus_name_copy(context, rrdset_context(st), PROMETHEUS_ELEMENT_MAX);
as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED);
homogeneous = 1;
@ -215,7 +215,7 @@ int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st)
homogeneous = 0;
} else {
if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE)
prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0);
prometheus_units_copy(units, rrdset_units(st), PROMETHEUS_ELEMENT_MAX, 0);
}
return 0;
@ -249,8 +249,8 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
D_EXPORTING,
"EXPORTING: not sending dimension '%s' of chart '%s' from host '%s', "
"its last data collection (%lu) is not within our timeframe (%lu to %lu)",
rd->id, rd->rrdset->id,
(host == localhost) ? instance->config.hostname : host->hostname,
rrddim_id(rd), rrdset_id(rd->rrdset),
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
(unsigned long)rd->last_collected_time.tv_sec,
(unsigned long)instance->after,
(unsigned long)instance->before);
@ -263,14 +263,14 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
prometheus_label_copy(
dimension,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
PROMETHEUS_ELEMENT_MAX);
snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix);
add_metric(
connector_specific_data->write_request,
name, chart, family, dimension,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
rd->last_collected_value, timeval_msec(&rd->last_collected_time));
} else {
// the dimensions of the chart, do not have the same algorithm, multiplier or divisor
@ -278,7 +278,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
prometheus_name_copy(
dimension,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
PROMETHEUS_ELEMENT_MAX);
snprintf(
name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", instance->config.prefix, context, dimension,
@ -287,7 +287,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
add_metric(
connector_specific_data->write_request,
name, chart, family, NULL,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
rd->last_collected_value, timeval_msec(&rd->last_collected_time));
}
} else {
@ -304,7 +304,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
prometheus_label_copy(
dimension,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
PROMETHEUS_ELEMENT_MAX);
snprintf(
name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", instance->config.prefix, context, units, suffix);
@ -312,7 +312,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
add_metric(
connector_specific_data->write_request,
name, chart, family, dimension,
(host == localhost) ? instance->config.hostname : host->hostname,
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host),
value, last_t * MSEC_PER_SEC);
}
}
@ -321,7 +321,9 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
return 0;
}
int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) {
static int format_variable_prometheus_remote_write_callback(const char *name_txt __maybe_unused, void *rv_ptr, void *data) {
RRDVAR *rv = rv_ptr;
struct prometheus_remote_write_variables_callback_options *opts = data;
if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) {
@ -335,12 +337,12 @@ int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) {
char name[PROMETHEUS_LABELS_MAX + 1];
char *suffix = "";
prometheus_name_copy(context, rv->name, PROMETHEUS_ELEMENT_MAX);
prometheus_name_copy(context, rrdvar_name(rv), PROMETHEUS_ELEMENT_MAX);
snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix);
NETDATA_DOUBLE value = rrdvar2number(rv);
add_variable(connector_specific_data->write_request, name,
(host == localhost) ? instance->config.hostname : host->hostname, value, opts->now / USEC_PER_MS);
(host == localhost) ? instance->config.hostname : rrdhost_hostname(host), value, opts->now / USEC_PER_MS);
}
return 0;
@ -361,7 +363,7 @@ int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST
.now = now_realtime_usec(),
};
return foreach_host_variable_callback(host, format_variable_prometheus_remote_write_callback, &opt);
return rrdvar_walkthrough_read(host->rrdvar_root_index, format_variable_prometheus_remote_write_callback, &opt);
}
/**

18
exporting/tests/exporting_fixtures.c
View File

@ -39,7 +39,7 @@ int setup_rrdhost()
localhost->rrd_update_every = 1;
localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2");
localhost->tags = string_strdupz("TAG1=VALUE1 TAG2=VALUE2");
localhost->host_labels = rrdlabels_create();
rrdlabels_add(localhost->host_labels, "key1", "value1", RRDLABEL_SRC_CONFIG);
@ -48,16 +48,16 @@ int setup_rrdhost()
localhost->rrdset_root = calloc(1, sizeof(RRDSET));
RRDSET *st = localhost->rrdset_root;
st->rrdhost = localhost;
strcpy(st->id, "chart_id");
st->name = strdupz("chart_name");
st->id = string_strdupz("chart_id");
st->name = string_strdupz("chart_name");
st->rrd_memory_mode |= RRD_MEMORY_MODE_SAVE;
st->update_every = 1;
localhost->rrdset_root->dimensions = calloc(1, sizeof(RRDDIM));
RRDDIM *rd = localhost->rrdset_root->dimensions;
rd->rrdset = st;
rd->id = strdupz("dimension_id");
rd->name = strdupz("dimension_name");
rd->id = string_strdupz("dimension_id");
rd->name = string_strdupz("dimension_name");
rd->last_collected_value = 123000321;
rd->last_collected_time.tv_sec = 15051;
rd->collections_counter++;
@ -77,18 +77,18 @@ int setup_rrdhost()
int teardown_rrdhost()
{
RRDDIM *rd = localhost->rrdset_root->dimensions;
free((void *)rd->name);
free((void *)rd->id);
string_freez(rd->name);
string_freez(rd->id);
free(rd->tiers[0]);
free(rd);
RRDSET *st = localhost->rrdset_root;
free((void *)st->name);
string_freez(st->name);
free(st);
rrdlabels_destroy(localhost->host_labels);
free((void *)localhost->tags);
string_freez(localhost->tags);
free(localhost);
return 0;

20
exporting/tests/netdata_doubles.c
View File

@ -177,12 +177,6 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
return RRD_MEMORY_MODE_NONE_NAME;
}
NETDATA_DOUBLE rrdvar2number(RRDVAR *rv)
{
(void)rv;
return 0;
}
int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data)
{
(void)host;
@ -256,6 +250,18 @@ void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, cha
(void)value;
}
void rrdcalc_update_rrdlabels(RRDSET *st) {
void rrdcalc_update_rrdlabels(RRDSET *st)
{
(void)st;
}
void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv)
{
(void)host;
(void)rv;
}
void db_execute(const char *cmd)
{
(void)cmd;
}

24
exporting/tests/test_exporting_engine.c
View File

@ -442,8 +442,8 @@ static void test_format_dimension_collected_json_plaintext(void **state)
assert_string_equal(
buffer_tostring(engine->instance_root->buffer),
"{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\","
"\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\","
"\"chart_context\":\"(null)\",\"chart_type\":\"(null)\",\"units\":\"(null)\",\"id\":\"dimension_id\","
"\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"\","
"\"chart_context\":\"\",\"chart_type\":\"\",\"units\":\"\",\"id\":\"dimension_id\","
"\"name\":\"dimension_name\",\"value\":123000321,\"timestamp\":15051}\n");
}
@ -459,8 +459,8 @@ static void test_format_dimension_stored_json_plaintext(void **state)
assert_string_equal(
buffer_tostring(engine->instance_root->buffer),
"{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\","
"\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\"," \
"\"chart_context\": \"(null)\",\"chart_type\":\"(null)\",\"units\": \"(null)\",\"id\":\"dimension_id\","
"\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"\"," \
"\"chart_context\": \"\",\"chart_type\":\"\",\"units\": \"\",\"id\":\"dimension_id\","
"\"name\":\"dimension_name\",\"value\":690565856.0000000,\"timestamp\": 15052}\n");
}
@ -1055,9 +1055,9 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
BUFFER *buffer = buffer_create(0);
localhost->hostname = strdupz("test_hostname");
localhost->rrdset_root->family = strdupz("test_family");
localhost->rrdset_root->context = strdupz("test_context");
localhost->hostname = string_strdupz("test_hostname");
localhost->rrdset_root->family = string_strdupz("test_family");
localhost->rrdset_root->context = string_strdupz("test_context");
expect_function_call(__wrap_now_realtime_sec);
will_return(__wrap_now_realtime_sec, 2);
@ -1069,7 +1069,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
assert_string_equal(
buffer_tostring(buffer),
"netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n"
"netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n"
"test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\"} 690565856.0000000\n");
buffer_flush(buffer);
@ -1085,7 +1085,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
assert_string_equal(
buffer_tostring(buffer),
"netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n"
"netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n"
"# TYPE test_prefix_test_context gauge\n"
"test_prefix_test_context{chart=\"chart_name\",family=\"test_family\",dimension=\"dimension_name\"} 690565856.0000000\n");
@ -1101,7 +1101,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
assert_string_equal(
buffer_tostring(buffer),
"netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n"
"netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n"
"test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\",instance=\"test_hostname\"} 690565856.0000000\n");
free(localhost->rrdset_root->context);
@ -1207,8 +1207,8 @@ static void test_format_host_prometheus_remote_write(void **state)
simple_connector_data->connector_specific_data = (void *)connector_specific_data;
connector_specific_data->write_request = (void *)0xff;
localhost->program_name = strdupz("test_program");
localhost->program_version = strdupz("test_version");
localhost->program_name = string_strdupz("test_program");
localhost->program_version = string_strdupz("test_version");
expect_function_call(__wrap_add_host_info);
expect_value(__wrap_add_host_info, write_request_p, 0xff);

1
exporting/tests/test_exporting_engine.h
View File

@ -4,6 +4,7 @@
#define TEST_EXPORTING_ENGINE_H 1
#include "libnetdata/libnetdata.h"
#include "database/rrdvar.h"
#include "exporting/exporting_engine.h"
#include "exporting/graphite/graphite.h"

250
health/health.c
View File

@ -155,18 +155,11 @@ static void health_reload_host(RRDHOST *host) {
// free all running alarms
rrdhost_wrlock(host);
while(host->templates)
rrdcalctemplate_unlink_and_free(host, host->templates);
while(host->alarms_templates)
rrdcalctemplate_unlink_and_free(host, host->alarms_templates);
RRDCALCTEMPLATE *rt,*next;
for(rt = host->alarms_template_with_foreach; rt ; rt = next) {
next = rt->next;
rrdcalctemplate_free(rt);
}
host->alarms_template_with_foreach = NULL;
while(host->alarms)
rrdcalc_unlink_and_free(host, host->alarms);
while(host->host_alarms)
rrdcalc_unlink_and_free(host, host->host_alarms);
RRDCALC *rc,*nc;
for(rc = host->alarms_with_foreach; rc ; rc = nc) {
@ -266,13 +259,13 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) {
// do not send notifications for internal statuses
debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
goto done;
}
if(unlikely(ae->new_status <= RRDCALC_STATUS_CLEAR && (ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) {
// do not send notifications for disabled statuses
debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)", ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
// mark it as run, so that we will send the same alarm if it happens again
goto done;
}
@ -292,7 +285,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
// we have executed this alarm notification in the past
if(t && t->new_status == ae->new_status) {
// don't send the notification for the same status again
debug(D_HEALTH, "Health not sending again notification for alarm '%s.%s' status %s", ae->chart, ae->name
debug(D_HEALTH, "Health not sending again notification for alarm '%s.%s' status %s", ae_chart_name(ae), ae_name(ae)
, rrdcalc_status2string(ae->new_status));
goto done;
}
@ -303,7 +296,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
if(unlikely(ae->new_status == RRDCALC_STATUS_CLEAR)) {
if((!(ae->flags & HEALTH_ENTRY_RUN_ONCE)) || (ae->flags & HEALTH_ENTRY_RUN_ONCE && ae->old_status < RRDCALC_STATUS_RAISED) ) {
debug(D_HEALTH, "Health not sending notification for first initialization of alarm '%s.%s' status %s"
, ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
, ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
goto done;
}
}
@ -312,14 +305,14 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
// Check if alarm notifications are silenced
if (ae->flags & HEALTH_ENTRY_FLAG_SILENCED) {
info("Health not sending notification for alarm '%s.%s' status %s (command API has disabled notifications)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
info("Health not sending notification for alarm '%s.%s' status %s (command API has disabled notifications)", ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
goto done;
}
static char command_to_run[ALARM_EXEC_COMMAND_LENGTH + 1];
const char *exec = (ae->exec) ? ae->exec : host->health_default_exec;
const char *recipient = (ae->recipient) ? ae->recipient : host->health_default_recipient;
const char *exec = (ae->exec) ? ae_exec(ae) : string2str(host->health_default_exec);
const char *recipient = (ae->recipient) ? ae_recipient(ae) : string2str(host->health_default_recipient);
int n_warn=0, n_crit=0;
RRDCALC *rc;
@ -330,13 +323,16 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
for(rc = host->alarms; rc && (n_warn + n_crit) < ACTIVE_ALARMS_LIST_EXAMINE ; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
continue;
if(unlikely((n_warn + n_crit) >= ACTIVE_ALARMS_LIST_EXAMINE))
break;
if (unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
if (likely(ae->alarm_id != rc->id) || likely(ae->alarm_event_id != rc->next_event_id - 1)) {
active_alerts[n_warn+n_crit].name = rc->name;
active_alerts[n_warn+n_crit].name = (char *)rrdcalc_name(rc);
active_alerts[n_warn+n_crit].last_status_change = rc->last_status_change;
active_alerts[n_warn+n_crit].status = rc->status;
n_warn++;
@ -344,7 +340,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
expr = rc->warning;
} else if (unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
if (likely(ae->alarm_id != rc->id) || likely(ae->alarm_event_id != rc->next_event_id - 1)) {
active_alerts[n_warn+n_crit].name = rc->name;
active_alerts[n_warn+n_crit].name = (char *)rrdcalc_name(rc);
active_alerts[n_warn+n_crit].last_status_change = rc->last_status_change;
active_alerts[n_warn+n_crit].status = rc->status;
n_crit++;
@ -379,39 +375,39 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
}
}
char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN");
char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");
snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" NETDATA_DOUBLE_FORMAT_ZERO
"' '" NETDATA_DOUBLE_FORMAT_ZERO
"' '%s' '%u' '%u' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d' '%s' '%s' '%s' '%s' '%s'",
exec,
recipient,
host->registry_hostname,
rrdhost_registry_hostname(host),
ae->unique_id,
ae->alarm_id,
ae->alarm_event_id,
(unsigned long)ae->when,
ae->name,
ae->chart?ae->chart:"NOCHART",
ae->family?ae->family:"NOFAMILY",
ae_name(ae),
ae->chart?ae_chart_name(ae):"NOCHART",
ae->family?ae_family(ae):"NOFAMILY",
rrdcalc_status2string(ae->new_status),
rrdcalc_status2string(ae->old_status),
ae->new_value,
ae->old_value,
ae->source?ae->source:"UNKNOWN",
ae->source?ae_source(ae):"UNKNOWN",
(uint32_t)ae->duration,
(uint32_t)ae->non_clear_duration,
ae->units?ae->units:"",
ae->info?ae->info:"",
ae->new_value_string,
ae->old_value_string,
ae_units(ae),
ae_info(ae),
ae_new_value_string(ae),
ae_old_value_string(ae),
(expr && expr->source)?expr->source:"NOSOURCE",
(expr && expr->error_msg)?buffer_tostring(expr->error_msg):"NOERRMSG",
n_warn,
n_crit,
buffer_tostring(warn_alarms),
buffer_tostring(crit_alarms),
ae->classification?ae->classification:"Unknown",
ae->classification?ae_classification(ae):"Unknown",
edit_command,
host != localhost ? host->machine_guid:""
);
@ -450,7 +446,7 @@ static inline void health_alarm_wait_for_execution(ALARM_ENTRY *ae) {
static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
debug(D_HEALTH, "Health alarm '%s.%s' = " NETDATA_DOUBLE_FORMAT_AUTO " - changed status from %s to %s",
ae->chart?ae->chart:"NOCHART", ae->name,
ae->chart?ae_chart_name(ae):"NOCHART", ae_name(ae),
ae->new_value,
rrdcalc_status2string(ae->old_status),
rrdcalc_status2string(ae->new_status)
@ -467,7 +463,7 @@ static inline void health_alarm_log_process(RRDHOST *host) {
ALARM_ENTRY *ae;
for(ae = host->health_log.alarms; ae && ae->unique_id >= host->health_last_processed_id; ae = ae->next) {
if(likely(!alarm_entry_isrepeating(host, ae))) {
if(likely(!(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING))) {
if(unlikely(
!(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) &&
!(ae->flags & HEALTH_ENTRY_FLAG_UPDATED)
@ -508,7 +504,7 @@ static inline void health_alarm_log_process(RRDHOST *host) {
ALARM_ENTRY *t = ae->next;
if(likely(!alarm_entry_isrepeating(host, ae))) {
if(likely(!(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING))) {
health_alarm_wait_for_execution(ae);
health_alarm_log_free_one_nochecks_nounlink(ae);
host->health_log.count--;
@ -522,7 +518,7 @@ static inline void health_alarm_log_process(RRDHOST *host) {
static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
if(unlikely(!rc->rrdset)) {
debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name);
debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rrdcalc_chart_name(rc), rrdcalc_name(rc));
return 0;
}
@ -533,27 +529,27 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
*next_run = rc->next_update;
}
debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now));
debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rrdcalc_chart_name(rc), rrdcalc_name(rc), (int) (rc->next_update - now));
return 0;
}
if(unlikely(!rc->update_every)) {
debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rc->chart?rc->chart:"NOCHART", rc->name);
debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rrdcalc_chart_name(rc), rrdcalc_name(rc));
return 0;
}
if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE))) {
debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as obsolete", rc->chart?rc->chart:"NOCHART", rc->name);
debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as obsolete", rrdcalc_chart_name(rc), rrdcalc_name(rc));
return 0;
}
if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_ARCHIVED))) {
debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as archived", rc->chart?rc->chart:"NOCHART", rc->name);
debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as archived", rrdcalc_chart_name(rc), rrdcalc_name(rc));
return 0;
}
if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name);
debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rrdcalc_chart_name(rc), rrdcalc_name(rc));
return 0;
}
@ -566,7 +562,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
if(unlikely(now + update_every < first /* || now - update_every > last */)) {
debug(D_HEALTH
, "Health not examining alarm '%s.%s' yet (wanted time is out of bounds - we need %lu but got %lu - %lu)."
, rc->chart ? rc->chart : "NOCHART", rc->name, (unsigned long) now, (unsigned long) first
, rrdcalc_chart_name(rc), rrdcalc_name(rc), (unsigned long) now, (unsigned long) first
, (unsigned long) last);
return 0;
}
@ -577,7 +573,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
if(needed + update_every < first || needed - update_every > last) {
debug(D_HEALTH
, "Health not examining alarm '%s.%s' yet (not enough data yet - we need %lu but got %lu - %lu)."
, rc->chart ? rc->chart : "NOCHART", rc->name, (unsigned long) needed, (unsigned long) first
, rrdcalc_chart_name(rc), rrdcalc_name(rc), (unsigned long) needed, (unsigned long) first
, (unsigned long) last);
return 0;
}
@ -614,30 +610,30 @@ static void health_main_cleanup(void *ptr) {
static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
}
static SILENCE_TYPE check_silenced(RRDCALC *rc, char* host, SILENCERS *silencers) {
static SILENCE_TYPE check_silenced(RRDCALC *rc, const char *host, SILENCERS *silencers) {
SILENCER *s;
debug(D_HEALTH, "Checking if alarm was silenced via the command API. Alarm info name:%s context:%s chart:%s host:%s family:%s",
rc->name, (rc->rrdset)?rc->rrdset->context:"", rc->chart, host, (rc->rrdset)?rc->rrdset->family:"");
rrdcalc_name(rc), (rc->rrdset)?rrdset_context(rc->rrdset):"", rrdcalc_chart_name(rc), host, (rc->rrdset)?rrdset_family(rc->rrdset):"");
for (s = silencers->silencers; s!=NULL; s=s->next){
if (
(!s->alarms_pattern || (rc->name && s->alarms_pattern && simple_pattern_matches(s->alarms_pattern,rc->name))) &&
(!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches(s->contexts_pattern,rc->rrdset->context))) &&
(!s->alarms_pattern || (rc->name && s->alarms_pattern && simple_pattern_matches(s->alarms_pattern, rrdcalc_name(rc)))) &&
(!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches(s->contexts_pattern, rrdset_context(rc->rrdset)))) &&
(!s->hosts_pattern || (host && s->hosts_pattern && simple_pattern_matches(s->hosts_pattern,host))) &&
(!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches(s->charts_pattern,rc->chart))) &&
(!s->families_pattern || (rc->rrdset && rc->rrdset->family && s->families_pattern && simple_pattern_matches(s->families_pattern,rc->rrdset->family)))
(!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches(s->charts_pattern, rrdcalc_chart_name(rc)))) &&
(!s->families_pattern || (rc->rrdset && rc->rrdset->family && s->families_pattern && simple_pattern_matches(s->families_pattern, rrdset_family(rc->rrdset))))
) {
debug(D_HEALTH, "Alarm matches command API silence entry %s:%s:%s:%s:%s", s->alarms,s->charts, s->contexts, s->hosts, s->families);
if (unlikely(silencers->stype == STYPE_NONE)) {
debug(D_HEALTH, "Alarm %s matched a silence entry, but no SILENCE or DISABLE command was issued via the command API. The match has no effect.", rc->name);
debug(D_HEALTH, "Alarm %s matched a silence entry, but no SILENCE or DISABLE command was issued via the command API. The match has no effect.", rrdcalc_name(rc));
} else {
debug(D_HEALTH, "Alarm %s via the command API - name:%s context:%s chart:%s host:%s family:%s"
, (silencers->stype == STYPE_DISABLE_ALARMS)?"Disabled":"Silenced"
, rc->name
, (rc->rrdset)?rc->rrdset->context:""
, rc->chart
, rrdcalc_name(rc)
, (rc->rrdset)?rrdset_context(rc->rrdset):""
, rrdcalc_chart_name(rc)
, host
, (rc->rrdset)?rc->rrdset->family:""
, (rc->rrdset)?rrdset_family(rc->rrdset):""
);
}
return silencers->stype;
@ -664,15 +660,15 @@ static int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) {
if (silencers->stype == STYPE_DISABLE_ALARMS) rc->rrdcalc_flags |= RRDCALC_FLAG_DISABLED;
else if (silencers->stype == STYPE_SILENCE_NOTIFICATIONS) rc->rrdcalc_flags |= RRDCALC_FLAG_SILENCED;
} else {
SILENCE_TYPE st = check_silenced(rc, host->hostname, silencers);
SILENCE_TYPE st = check_silenced(rc, rrdhost_hostname(host), silencers);
if (st == STYPE_DISABLE_ALARMS) rc->rrdcalc_flags |= RRDCALC_FLAG_DISABLED;
else if (st == STYPE_SILENCE_NOTIFICATIONS) rc->rrdcalc_flags |= RRDCALC_FLAG_SILENCED;
}
if (rrdcalc_flags_old != rc->rrdcalc_flags) {
info("Alarm silencing changed for host '%s' alarm '%s': Disabled %s->%s Silenced %s->%s",
host->hostname,
rc->name,
rrdhost_hostname(host),
rrdcalc_name(rc),
(rrdcalc_flags_old & RRDCALC_FLAG_DISABLED)?"true":"false",
(rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false",
(rrdcalc_flags_old & RRDCALC_FLAG_SILENCED)?"true":"false",
@ -812,7 +808,7 @@ void *health_main(void *ptr) {
info(
"Postponing health checks for %"PRId64" seconds, on host '%s'.",
(int64_t)hibernation_delay,
host->hostname);
rrdhost_hostname(host));
host->health_delay_up_to = now + hibernation_delay;
}
@ -821,7 +817,7 @@ void *health_main(void *ptr) {
if (unlikely(now < host->health_delay_up_to))
continue;
info("Resuming health checks on host '%s'.", host->hostname);
info("Resuming health checks on host '%s'.", rrdhost_hostname(host));
host->health_delay_up_to = 0;
}
@ -839,7 +835,7 @@ void *health_main(void *ptr) {
rrdhost_rdlock(host);
// the first loop is to lookup values from the db
for (rc = host->alarms; rc; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
if (update_disabled_silenced(host, rc))
continue;
@ -852,10 +848,33 @@ void *health_main(void *ptr) {
if (!rrdcalc_isrepeating(rc)) {
worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
time_t now = now_realtime_sec();
ALARM_ENTRY *ae = health_create_alarm_entry(
host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id, rc->rrdset->context,
rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
rc->value, NAN, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info, 0, 0);
host,
rc->id,
rc->next_event_id++,
rc->config_hash_id,
now,
rc->name,
rc->rrdset->id,
rc->rrdset->context,
rc->rrdset->family,
rc->classification,
rc->component,
rc->type,
rc->exec,
rc->recipient,
now - rc->last_status_change,
rc->value,
NAN,
rc->status,
RRDCALC_STATUS_REMOVED,
rc->source,
rc->units,
rc->info,
0,
rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
if (ae) {
health_alarm_log(host, ae);
rc->old_status = rc->status;
@ -890,7 +909,7 @@ void *health_main(void *ptr) {
/* time_t old_db_timestamp = rc->db_before; */
int value_is_null = 0;
int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rc->dimensions, 1,
int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
rc->after, rc->before, rc->group, NULL,
0, rc->options,
&rc->db_after,&rc->db_before,
@ -903,7 +922,7 @@ void *health_main(void *ptr) {
rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, ret
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
);
} else
rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR;
@ -930,13 +949,13 @@ void *health_main(void *ptr) {
debug(D_HEALTH,
"Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
);
} else
rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT, host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
rc->value
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
);
}
@ -952,7 +971,7 @@ void *health_main(void *ptr) {
rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",
host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)
);
} else {
@ -960,9 +979,9 @@ void *health_main(void *ptr) {
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
rc->calculation->parsed_as, rc->calculation->result,
buffer_tostring(rc->calculation->error_msg), rc->source
buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc)
);
rc->value = rc->calculation->result;
@ -980,7 +999,7 @@ void *health_main(void *ptr) {
if (unlikely(runnable && !netdata_exit)) {
rrdhost_rdlock(host);
for (rc = host->alarms; rc; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE)))
continue;
@ -1002,15 +1021,15 @@ void *health_main(void *ptr) {
debug(D_HEALTH,
"Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",
host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
buffer_tostring(rc->warning->error_msg)
);
} else {
rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART",
rc->name, rc->warning->result, buffer_tostring(rc->warning->error_msg), rc->source
": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc)
);
warning_status = rrdcalc_value2status(rc->warning->result);
}
@ -1028,16 +1047,16 @@ void *health_main(void *ptr) {
debug(D_HEALTH,
"Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",
host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
buffer_tostring(rc->critical->error_msg)
);
} else {
rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART",
rc->name, rc->critical->result, buffer_tostring(rc->critical->error_msg),
rc->source
": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg),
rrdcalc_source(rc)
);
critical_status = rrdcalc_value2status(rc->critical->result);
}
@ -1113,15 +1132,36 @@ void *health_main(void *ptr) {
ALARM_ENTRY *ae = health_create_alarm_entry(
host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id, rc->rrdset->context,
rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info,
rc->delay_last,
(
((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0)
host,
rc->id,
rc->next_event_id++,
rc->config_hash_id,
now,
rc->name,
rc->rrdset->id,
rc->rrdset->context,
rc->rrdset->family,
rc->classification,
rc->component,
rc->type,
rc->exec,
rc->recipient,
now - rc->last_status_change,
rc->old_value,
rc->value,
rc->status,
status,
rc->source,
rc->units,
rc->info,
rc->delay_last,
(
((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
(rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
)
);
health_alarm_log(host, ae);
rc->last_status_change = now;
@ -1138,7 +1178,7 @@ void *health_main(void *ptr) {
// process repeating alarms
RRDCALC *rc;
for(rc = host->alarms; rc ; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
int repeat_every = 0;
if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
@ -1164,16 +1204,38 @@ void *health_main(void *ptr) {
worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
rc->last_repeat = now;
if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
ALARM_ENTRY *ae = health_create_alarm_entry(
host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id, rc->rrdset->context,
rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
rc->old_value, rc->value, rc->old_status, rc->status, rc->source, rc->units, rc->info,
rc->delay_last,
(
((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0)
host,
rc->id,
rc->next_event_id++,
rc->config_hash_id,
now,
rc->name,
rc->rrdset->id,
rc->rrdset->context,
rc->rrdset->family,
rc->classification,
rc->component,
rc->type,
rc->exec,
rc->recipient,
now - rc->last_status_change,
rc->old_value,
rc->value,
rc->old_status,
rc->status,
rc->source,
rc->units,
rc->info,
rc->delay_last,
(
((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
(rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
)
);
ae->last_repeat = rc->last_repeat;
if (!(rc->rrdcalc_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
ae->flags |= HEALTH_ENTRY_RUN_ONCE;

52
health/health.h
View File

@ -14,6 +14,7 @@ extern unsigned int default_health_enabled;
#define HEALTH_ENTRY_FLAG_SILENCED 0x00000010
#define HEALTH_ENTRY_RUN_ONCE 0x00000020
#define HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS 0x00000040
#define HEALTH_ENTRY_FLAG_IS_REPEATING 0x00000080
#define HEALTH_ENTRY_FLAG_SAVED 0x10000000
#define HEALTH_ENTRY_FLAG_ACLK_QUEUED 0x20000000
@ -35,7 +36,6 @@ extern void health_init(void);
extern void health_reload(void);
extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, NETDATA_DOUBLE *result);
extern void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* context, RRDCALC_STATUS status);
extern void health_alarms2json(RRDHOST *host, BUFFER *wb, int all);
extern void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all);
@ -49,30 +49,30 @@ extern void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
extern void health_alarm_log_load(RRDHOST *host);
extern ALARM_ENTRY* health_create_alarm_entry(
RRDHOST *host,
uint32_t alarm_id,
uint32_t alarm_event_id,
uuid_t config_hash_id,
time_t when,
const char *name,
const char *chart,
const char *chart_context,
const char *family,
const char *classification,
const char *component,
const char *type,
const char *exec,
const char *recipient,
time_t duration,
NETDATA_DOUBLE old_value,
NETDATA_DOUBLE new_value,
RRDCALC_STATUS old_status,
RRDCALC_STATUS new_status,
const char *source,
const char *units,
const char *info,
int delay,
uint32_t flags);
RRDHOST *host,
uint32_t alarm_id,
uint32_t alarm_event_id,
const uuid_t config_hash_id,
time_t when,
STRING *name,
STRING *chart,
STRING *chart_context,
STRING *family,
STRING *classification,
STRING *component,
STRING *type,
STRING *exec,
STRING *recipient,
time_t duration,
NETDATA_DOUBLE old_value,
NETDATA_DOUBLE new_value,
RRDCALC_STATUS old_status,
RRDCALC_STATUS new_status,
STRING *source,
STRING *units,
STRING *info,
int delay,
uint32_t flags);
extern void health_alarm_log(RRDHOST *host, ALARM_ENTRY *ae);
@ -90,6 +90,6 @@ extern void health_label_log_save(RRDHOST *host);
extern char *health_edit_command_from_source(const char *source);
extern void sql_refresh_hashes(void);
extern SIMPLE_PATTERN *health_pattern_from_foreach(char *s);
extern SIMPLE_PATTERN *health_pattern_from_foreach(const char *s);
#endif //NETDATA_HEALTH_H

632
health/health_config.c
View File

File diff suppressed because it is too large Load Diff

97
health/health_json.c
View File

@ -14,7 +14,7 @@ void health_string2json(BUFFER *wb, const char *prefix, const char *label, const
}
void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN");
char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");
char config_hash_id[GUID_LEN + 1];
uuid_unparse_lower(ae->config_hash_id, config_hash_id);
@ -57,30 +57,30 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
"\t\t\"old_value_string\": \"%s\",\n"
"\t\t\"last_repeat\": \"%lu\",\n"
"\t\t\"silenced\": \"%s\",\n"
, host->hostname
, rrdhost_hostname(host)
, host->utc_offset
, host->abbrev_timezone
, rrdhost_abbrev_timezone(host)
, ae->unique_id
, ae->alarm_id
, ae->alarm_event_id
, config_hash_id
, ae->name
, ae->chart
, ae->chart_context
, ae->family
, ae->classification?ae->classification:"Unknown"
, ae->component?ae->component:"Unknown"
, ae->type?ae->type:"Unknown"
, ae_name(ae)
, ae_chart_name(ae)
, ae_chart_context(ae)
, ae_family(ae)
, ae->classification?ae_classification(ae):"Unknown"
, ae->component?ae_component(ae):"Unknown"
, ae->type?ae_type(ae):"Unknown"
, (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false"
, (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false"
, (unsigned long)ae->exec_run_timestamp
, (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false"
, ae->exec?ae->exec:host->health_default_exec
, ae->recipient?ae->recipient:host->health_default_recipient
, ae->exec?ae_exec(ae):string2str(host->health_default_exec)
, ae->recipient?ae_recipient(ae):string2str(host->health_default_recipient)
, ae->exec_code
, ae->source
, ae_source(ae)
, edit_command
, ae->units?ae->units:""
, ae_units(ae)
, (unsigned long)ae->when
, (unsigned long)ae->duration
, (unsigned long)ae->non_clear_duration
@ -90,13 +90,13 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
, (unsigned long)ae->delay_up_to_timestamp
, ae->updated_by_id
, ae->updates_id
, ae->new_value_string
, ae->old_value_string
, ae_new_value_string(ae)
, ae_old_value_string(ae)
, (unsigned long)ae->last_repeat
, (ae->flags & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false"
);
health_string2json(wb, "\t\t", "info", ae->info ? ae->info : "", ",\n");
health_string2json(wb, "\t\t", "info", ae->info ? ae_info(ae) : "", ",\n");
if(unlikely(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {
buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n");
@ -122,11 +122,12 @@ void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *char
unsigned int max = host->health_log.max;
unsigned int count = 0;
uint32_t hash_chart = 0;
if (chart) hash_chart = simple_hash(chart);
STRING *chart_string = string_strdupz(chart);
ALARM_ENTRY *ae;
for (ae = host->health_log.alarms; ae && count < max; ae = ae->next) {
if ((ae->unique_id > after) && (!chart || (ae->hash_chart == hash_chart && !strcmp(ae->chart, chart)))) {
if ((ae->unique_id > after) && (!chart || chart_string == ae->chart)) {
if (likely(count))
buffer_strcat(wb, ",");
health_alarm_entry2json_nolock(wb, ae, host);
@ -134,6 +135,8 @@ void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *char
}
}
string_freez(chart_string);
buffer_strcat(wb, "\n]\n");
netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);
@ -144,7 +147,7 @@ static inline void health_rrdcalc_values2json_nolock(RRDHOST *host, BUFFER *wb,
buffer_sprintf(wb,
"\t\t\"%s.%s\": {\n"
"\t\t\t\"id\": %lu,\n"
, rc->chart, rc->name
, rrdcalc_chart_name(rc), rrdcalc_name(rc)
, (unsigned long)rc->id);
buffer_strcat(wb, "\t\t\t\"value\":");
@ -164,7 +167,7 @@ static inline void health_rrdcalc_values2json_nolock(RRDHOST *host, BUFFER *wb,
static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC *rc) {
char value_string[100 + 1];
format_value_and_unit(value_string, 100, rc->value, rc->units, -1);
format_value_and_unit(value_string, 100, rc->value, rrdcalc_units(rc), -1);
char hash_id[GUID_LEN + 1];
uuid_unparse_lower(rc->config_hash_id, hash_id);
@ -203,23 +206,23 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
"\t\t\t\"value_string\": \"%s\",\n"
"\t\t\t\"last_repeat\": \"%lu\",\n"
"\t\t\t\"times_repeat\": %lu,\n"
, rc->chart, rc->name
, rrdcalc_chart_name(rc), rrdcalc_name(rc)
, (unsigned long)rc->id
, hash_id
, rc->name
, rc->chart
, (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
, rc->classification?rc->classification:"Unknown"
, rc->component?rc->component:"Unknown"
, rc->type?rc->type:"Unknown"
, rrdcalc_name(rc)
, rrdcalc_chart_name(rc)
, (rc->rrdset)?rrdset_family(rc->rrdset):""
, rc->classification?rrdcalc_classification(rc):"Unknown"
, rc->component?rrdcalc_component(rc):"Unknown"
, rc->type?rrdcalc_type(rc):"Unknown"
, (rc->rrdset)?"true":"false"
, (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false"
, (rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)?"true":"false"
, rc->exec?rc->exec:host->health_default_exec
, rc->recipient?rc->recipient:host->health_default_recipient
, rc->source
, rc->units?rc->units:""
, rc->info?rc->info:""
, rc->exec?rrdcalc_exec(rc):string2str(host->health_default_exec)
, rc->recipient?rrdcalc_recipient(rc):string2str(host->health_default_recipient)
, rrdcalc_source(rc)
, rrdcalc_units(rc)
, rrdcalc_info(rc)
, rrdcalc_status2string(rc->status)
, (unsigned long)rc->last_status_change
, (unsigned long)rc->last_updated
@ -243,8 +246,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
}
if(RRDCALC_HAS_DB_LOOKUP(rc)) {
if(rc->dimensions && *rc->dimensions)
health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n");
if(rc->dimensions)
health_string2json(wb, "\t\t\t", "lookup_dimensions", rrdcalc_dimensions(rc), ",\n");
buffer_sprintf(wb,
"\t\t\t\"db_after\": %lu,\n"
@ -310,20 +313,24 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL
while(p && *p && (tok = mystrsep(&p, ", |"))) {
if(!*tok) continue;
for(rc = host->alarms; rc ; rc = rc->next) {
STRING *tok_string = string_strdupz(tok);
foreach_rrdcalc_in_rrdhost(host, rc) {
if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
continue;
if (unlikely(!rrdset_is_available_for_exporting_and_alarms(rc->rrdset)))
continue;
if(unlikely(rc->rrdset && rc->rrdset->hash_context == simple_hash(tok)
&& !strcmp(rc->rrdset->context, tok)
&& ((status==RRDCALC_STATUS_RAISED)?(rc->status >= RRDCALC_STATUS_WARNING):rc->status == status)))
if(unlikely(rc->rrdset
&& rc->rrdset->context == tok_string
&& ((status==RRDCALC_STATUS_RAISED)?(rc->status >= RRDCALC_STATUS_WARNING):rc->status == status)))
numberOfAlarms++;
}
string_freez(tok_string);
}
}
else {
for(rc = host->alarms; rc ; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
continue;
if (unlikely(!rrdset_is_available_for_exporting_and_alarms(rc->rrdset)))
@ -339,8 +346,8 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL
static void health_alarms2json_fill_alarms(RRDHOST *host, BUFFER *wb, int all, void (*fp)(RRDHOST *, BUFFER *, RRDCALC *)) {
RRDCALC *rc;
int i;
for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
int i = 0;
foreach_rrdcalc_in_rrdhost(host, rc) {
if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
continue;
@ -363,7 +370,7 @@ void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
"\n\t\"status\": %s,"
"\n\t\"now\": %lu,"
"\n\t\"alarms\": {\n",
host->hostname,
rrdhost_hostname(host),
(host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0,
host->health_enabled?"true":"false",
(unsigned long)now_realtime_sec());
@ -383,7 +390,7 @@ void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all) {
rrdhost_rdlock(host);
buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\","
"\n\t\"alarms\": {\n",
host->hostname);
rrdhost_hostname(host));
health_alarms2json_fill_alarms(host, wb, all, health_rrdcalc_values2json_nolock);

292
health/health_log.c
View File

@ -14,11 +14,11 @@ inline int health_alarm_log_open(RRDHOST *host) {
if(host->health_log_fp) {
if (setvbuf(host->health_log_fp, NULL, _IOLBF, 0) != 0)
error("HEALTH [%s]: cannot set line buffering on health log file '%s'.", host->hostname, host->health_log_filename);
error("HEALTH [%s]: cannot set line buffering on health log file '%s'.", rrdhost_hostname(host), host->health_log_filename);
return 0;
}
error("HEALTH [%s]: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", host->hostname, host->health_log_filename);
error("HEALTH [%s]: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", rrdhost_hostname(host), host->health_log_filename);
return -1;
}
@ -45,13 +45,13 @@ static inline void health_log_rotate(RRDHOST *host) {
snprintfz(old_filename, FILENAME_MAX, "%s.old", host->health_log_filename);
if(unlink(old_filename) == -1 && errno != ENOENT)
error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, old_filename);
error("HEALTH [%s]: cannot remove old alarms log file '%s'", rrdhost_hostname(host), old_filename);
if(link(host->health_log_filename, old_filename) == -1 && errno != ENOENT)
error("HEALTH [%s]: cannot move file '%s' to '%s'.", host->hostname, host->health_log_filename, old_filename);
error("HEALTH [%s]: cannot move file '%s' to '%s'.", rrdhost_hostname(host), host->health_log_filename, old_filename);
if(unlink(host->health_log_filename) == -1 && errno != ENOENT)
error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, host->health_log_filename);
error("HEALTH [%s]: cannot remove old alarms log file '%s'", rrdhost_hostname(host), host->health_log_filename);
// open it with truncate
host->health_log_fp = fopen(host->health_log_filename, "w");
@ -59,7 +59,7 @@ static inline void health_log_rotate(RRDHOST *host) {
if(host->health_log_fp)
fclose(host->health_log_fp);
else
error("HEALTH [%s]: cannot truncate health log '%s'", host->hostname, host->health_log_filename);
error("HEALTH [%s]: cannot truncate health log '%s'", rrdhost_hostname(host), host->health_log_filename);
host->health_log_fp = NULL;
@ -80,7 +80,7 @@ inline void health_label_log_save(RRDHOST *host) {
if (unlikely(fprintf(host->health_log_fp, "L\t%s", write) < 0))
error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.",
host->hostname, host->health_log_filename);
rrdhost_hostname(host), host->health_log_filename);
else
host->health_log_entries_written++;
@ -103,7 +103,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
"\t%s\t%s\t%s"
"\n"
, (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'
, host->hostname
, rrdhost_hostname(host)
, ae->unique_id
, ae->alarm_id
@ -118,14 +118,14 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
, (uint32_t)ae->exec_run_timestamp
, (uint32_t)ae->delay_up_to_timestamp
, (ae->name)?ae->name:""
, (ae->chart)?ae->chart:""
, (ae->family)?ae->family:""
, (ae->exec)?ae->exec:""
, (ae->recipient)?ae->recipient:""
, (ae->source)?ae->source:""
, (ae->units)?ae->units:""
, (ae->info)?ae->info:""
, ae_name(ae)
, ae_chart_name(ae)
, ae_family(ae)
, ae_exec(ae)
, ae_recipient(ae)
, ae_source(ae)
, ae_units(ae)
, ae_info(ae)
, ae->exec_code
, ae->new_status
@ -135,11 +135,11 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
, ae->new_value
, ae->old_value
, (uint64_t)ae->last_repeat
, (ae->classification)?ae->classification:"Unknown"
, (ae->component)?ae->component:"Unknown"
, (ae->type)?ae->type:"Unknown"
, (ae->classification)?ae_classification(ae):"Unknown"
, (ae->component)?ae_component(ae):"Unknown"
, (ae->type)?ae_type(ae):"Unknown"
) < 0))
error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", host->hostname, host->health_log_filename);
error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", rrdhost_hostname(host), host->health_log_filename);
else {
ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
host->health_log_entries_written++;
@ -156,18 +156,23 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
static uint32_t is_valid_alarm_id(RRDHOST *host, const char *chart, const char *name, uint32_t alarm_id)
{
uint32_t hash_chart = simple_hash(chart);
uint32_t hash_name = simple_hash(name);
STRING *chart_string = string_strdupz(chart);
STRING *name_string = string_strdupz(name);
uint32_t ret = 1;
ALARM_ENTRY *ae;
for(ae = host->health_log.alarms; ae ;ae = ae->next) {
if (unlikely(
ae->alarm_id == alarm_id && (!(ae->hash_name == hash_name && ae->hash_chart == hash_chart &&
!strcmp(name, ae->name) && !strcmp(chart, ae->chart))))) {
return 0;
if (unlikely(ae->alarm_id == alarm_id && (!(chart_string == ae->chart && name_string == ae->name)))) {
ret = 0;
break;
}
}
return 1;
string_freez(chart_string);
string_freez(name_string);
return ret;
}
static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) {
@ -179,6 +184,11 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
DICTIONARY *all_rrdcalcs = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
RRDCALC *rc;
foreach_rrdcalc_in_rrdhost(host, rc)
dictionary_set(all_rrdcalcs, rrdcalc_name(rc), rc, sizeof(*rc));
while((s = fgets_trim_len(buf, 65536, fp, &len))) {
host->health_log_entries_written++;
line++;
@ -192,7 +202,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
*s = '\0';
pointers[entries++] = ++s;
if(entries >= max_entries) {
error("HEALTH [%s]: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", host->hostname, line, filename, max_entries);
error("HEALTH [%s]: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", rrdhost_hostname(host), line, filename, max_entries);
break;
}
}
@ -206,7 +216,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
ALARM_ENTRY *ae = NULL;
if(entries < 27) {
error("HEALTH [%s]: line %zu of file '%s' should have at least 27 entries, but it has %d. Ignoring it.", host->hostname, line, filename, entries);
error("HEALTH [%s]: line %zu of file '%s' should have at least 27 entries, but it has %d. Ignoring it.", rrdhost_hostname(host), line, filename, entries);
errored++;
continue;
}
@ -214,14 +224,14 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
// check that we have valid ids
uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
if(!unique_id) {
error("HEALTH [%s]: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", host->hostname, line, filename, unique_id, pointers[2]);
error("HEALTH [%s]: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", rrdhost_hostname(host), line, filename, unique_id, pointers[2]);
errored++;
continue;
}
uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
if(!alarm_id) {
error("HEALTH [%s]: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", host->hostname, line, filename, alarm_id, pointers[3]);
error("HEALTH [%s]: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", rrdhost_hostname(host), line, filename, alarm_id, pointers[3]);
errored++;
continue;
}
@ -232,18 +242,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
char* alarm_name = pointers[13];
last_repeat = (time_t)strtoul(pointers[27], NULL, 16);
RRDCALC *rc = alarm_max_last_repeat(host, alarm_name,simple_hash(alarm_name));
if (!rc) {
for(rc = host->alarms; rc ; rc = rc->next) {
RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_name, (avl_t *)rc);
if(rdcmp != rc) {
error("Cannot insert the alarm index ID using log %s", rc->name);
}
}
rc = alarm_max_last_repeat(host, alarm_name,simple_hash(alarm_name));
}
rc = dictionary_get(all_rrdcalcs, alarm_name);
if(unlikely(rc)) {
if (rrdcalc_isrepeating(rc)) {
rc->last_repeat = last_repeat;
@ -259,7 +258,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
// make sure it is properly numbered
if(unlikely(host->health_log.alarms && unique_id < host->health_log.alarms->unique_id)) {
error( "HEALTH [%s]: line %zu of file '%s' has alarm log entry %u in wrong order. Ignoring it."
, host->hostname, line, filename, unique_id);
, rrdhost_hostname(host), line, filename, unique_id);
errored++;
continue;
}
@ -272,7 +271,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
if(unlikely(unique_id == ae->unique_id)) {
if(unlikely(*pointers[0] == 'A')) {
error("HEALTH [%s]: line %zu of file '%s' adds duplicate alarm log entry %u. Using the later."
, host->hostname, line, filename, unique_id);
, rrdhost_hostname(host), line, filename, unique_id);
*pointers[0] = 'U';
duplicate++;
}
@ -298,8 +297,13 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
// error("HEALTH [%s]: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", host->hostname, line, filename, pointers[1], host->hostname);
ae->unique_id = unique_id;
if (!is_valid_alarm_id(host, pointers[14], pointers[13], alarm_id))
alarm_id = rrdcalc_get_unique_id(host, pointers[14], pointers[13], NULL);
if (!is_valid_alarm_id(host, pointers[14], pointers[13], alarm_id)) {
STRING *chart = string_strdupz(pointers[14]);
STRING *name = string_strdupz(pointers[13]);
alarm_id = rrdcalc_get_unique_id(host, chart, name, NULL);
string_freez(chart);
string_freez(name);
}
ae->alarm_id = alarm_id;
ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16);
ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16);
@ -315,36 +319,29 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16);
ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16);
freez(ae->name);
ae->name = strdupz(pointers[13]);
ae->hash_name = simple_hash(ae->name);
string_freez(ae->name);
ae->name = string_strdupz(pointers[13]);
freez(ae->chart);
ae->chart = strdupz(pointers[14]);
ae->hash_chart = simple_hash(ae->chart);
string_freez(ae->chart);
ae->chart = string_strdupz(pointers[14]);
freez(ae->family);
ae->family = strdupz(pointers[15]);
string_freez(ae->family);
ae->family = string_strdupz(pointers[15]);
freez(ae->exec);
ae->exec = strdupz(pointers[16]);
if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
string_freez(ae->exec);
ae->exec = string_strdupz(pointers[16]);
freez(ae->recipient);
ae->recipient = strdupz(pointers[17]);
if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
string_freez(ae->recipient);
ae->recipient = string_strdupz(pointers[17]);
freez(ae->source);
ae->source = strdupz(pointers[18]);
if(!*ae->source) { freez(ae->source); ae->source = NULL; }
string_freez(ae->source);
ae->source = string_strdupz(pointers[18]);
freez(ae->units);
ae->units = strdupz(pointers[19]);
if(!*ae->units) { freez(ae->units); ae->units = NULL; }
string_freez(ae->units);
ae->units = string_strdupz(pointers[19]);
freez(ae->info);
ae->info = strdupz(pointers[20]);
if(!*ae->info) { freez(ae->info); ae->info = NULL; }
string_freez(ae->info);
ae->info = string_strdupz(pointers[20]);
ae->exec_code = str2i(pointers[21]);
ae->new_status = str2i(pointers[22]);
@ -357,24 +354,21 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
ae->last_repeat = last_repeat;
if (likely(entries > 30)) {
freez(ae->classification);
ae->classification = strdupz(pointers[28]);
if(!*ae->classification) { freez(ae->classification); ae->classification = NULL; }
string_freez(ae->classification);
ae->classification = string_strdupz(pointers[28]);
freez(ae->component);
ae->component = strdupz(pointers[29]);
if(!*ae->component) { freez(ae->component); ae->component = NULL; }
string_freez(ae->component);
ae->component = string_strdupz(pointers[29]);
freez(ae->type);
ae->type = strdupz(pointers[30]);
if(!*ae->type) { freez(ae->type); ae->type = NULL; }
string_freez(ae->type);
ae->type = string_strdupz(pointers[30]);
}
char value_string[100 + 1];
freez(ae->old_value_string);
freez(ae->new_value_string);
ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
string_freez(ae->old_value_string);
string_freez(ae->new_value_string);
ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1));
ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1));
// add it to host if not already there
if(unlikely(*pointers[0] == 'A')) {
@ -395,11 +389,14 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
host->health_max_alarm_id = ae->alarm_id;
}
else {
error("HEALTH [%s]: line %zu of file '%s' is invalid (unrecognized entry type '%s').", host->hostname, line, filename, pointers[0]);
error("HEALTH [%s]: line %zu of file '%s' is invalid (unrecognized entry type '%s').", rrdhost_hostname(host), line, filename, pointers[0]);
errored++;
}
}
dictionary_destroy(all_rrdcalcs);
all_rrdcalcs = NULL;
netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);
freez(buf);
@ -411,7 +408,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
if (unlikely(!host->health_log.next_alarm_id || host->health_log.next_alarm_id <= host->health_max_alarm_id))
host->health_log.next_alarm_id = host->health_max_alarm_id + 1;
debug(D_HEALTH, "HEALTH [%s]: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", host->hostname, filename, loaded, updated, errored, duplicate);
debug(D_HEALTH, "HEALTH [%s]: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", rrdhost_hostname(host), filename, loaded, updated, errored, duplicate);
return loaded;
}
@ -422,7 +419,7 @@ inline void health_alarm_log_load(RRDHOST *host) {
snprintfz(filename, FILENAME_MAX, "%s.old", host->health_log_filename);
FILE *fp = fopen(filename, "r");
if(!fp)
error("HEALTH [%s]: cannot open health file: %s", host->hostname, filename);
error("HEALTH [%s]: cannot open health file: %s", rrdhost_hostname(host), filename);
else {
health_alarm_log_read(host, fp, filename);
fclose(fp);
@ -431,7 +428,7 @@ inline void health_alarm_log_load(RRDHOST *host) {
host->health_log_entries_written = 0;
fp = fopen(host->health_log_filename, "r");
if(!fp)
error("HEALTH [%s]: cannot open health file: %s", host->hostname, host->health_log_filename);
error("HEALTH [%s]: cannot open health file: %s", rrdhost_hostname(host), host->health_log_filename);
else {
health_alarm_log_read(host, fp, host->health_log_filename);
fclose(fp);
@ -443,63 +440,48 @@ inline void health_alarm_log_load(RRDHOST *host) {
// health alarm log management
inline ALARM_ENTRY* health_create_alarm_entry(
RRDHOST *host,
uint32_t alarm_id,
uint32_t alarm_event_id,
uuid_t config_hash_id,
time_t when,
const char *name,
const char *chart,
const char *chart_context,
const char *family,
const char *class,
const char *component,
const char *type,
const char *exec,
const char *recipient,
time_t duration,
NETDATA_DOUBLE old_value,
NETDATA_DOUBLE new_value,
RRDCALC_STATUS old_status,
RRDCALC_STATUS new_status,
const char *source,
const char *units,
const char *info,
int delay,
uint32_t flags
RRDHOST *host,
uint32_t alarm_id,
uint32_t alarm_event_id,
const uuid_t config_hash_id,
time_t when,
STRING *name,
STRING *chart,
STRING *chart_context,
STRING *family,
STRING *class,
STRING *component,
STRING *type,
STRING *exec,
STRING *recipient,
time_t duration,
NETDATA_DOUBLE old_value,
NETDATA_DOUBLE new_value,
RRDCALC_STATUS old_status,
RRDCALC_STATUS new_status,
STRING *source,
STRING *units,
STRING *info,
int delay,
uint32_t flags
) {
debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id);
ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
ae->name = strdupz(name);
ae->hash_name = simple_hash(ae->name);
if(chart) {
ae->chart = strdupz(chart);
ae->hash_chart = simple_hash(ae->chart);
}
if(chart_context)
ae->chart_context = strdupz(chart_context);
ae->name = string_dup(name);
ae->chart = string_dup(chart);
ae->chart_context = string_dup(chart_context);
uuid_copy(ae->config_hash_id, *((uuid_t *) config_hash_id));
if(family)
ae->family = strdupz(family);
if (class)
ae->classification = strdupz(class);
if (component)
ae->component = strdupz(component);
if (type)
ae->type = strdupz(type);
if(exec) ae->exec = strdupz(exec);
if(recipient) ae->recipient = strdupz(recipient);
if(source) ae->source = strdupz(source);
if(units) ae->units = strdupz(units);
ae->family = string_dup(family);
ae->classification = string_dup(class);
ae->component = string_dup(component);
ae->type = string_dup(type);
ae->exec = string_dup(exec);
ae->recipient = string_dup(recipient);
ae->source = string_dup(source);
ae->units = string_dup(units);
ae->unique_id = host->health_log.next_log_id++;
ae->alarm_id = alarm_id;
@ -509,12 +491,10 @@ inline ALARM_ENTRY* health_create_alarm_entry(
ae->new_value = new_value;
char value_string[100 + 1];
ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
if (info)
ae->info = strdupz(info);
ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1));
ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1));
ae->info = string_dup(info);
ae->old_status = old_status;
ae->new_status = new_status;
ae->duration = duration;
@ -570,20 +550,20 @@ inline void health_alarm_log(
}
inline void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae) {
freez(ae->name);
freez(ae->chart);
freez(ae->chart_context);
freez(ae->family);
freez(ae->classification);
freez(ae->component);
freez(ae->type);
freez(ae->exec);
freez(ae->recipient);
freez(ae->source);
freez(ae->units);
freez(ae->info);
freez(ae->old_value_string);
freez(ae->new_value_string);
string_freez(ae->name);
string_freez(ae->chart);
string_freez(ae->chart_context);
string_freez(ae->family);
string_freez(ae->classification);
string_freez(ae->component);
string_freez(ae->type);
string_freez(ae->exec);
string_freez(ae->recipient);
string_freez(ae->source);
string_freez(ae->units);
string_freez(ae->info);
string_freez(ae->old_value_string);
string_freez(ae->new_value_string);
freez(ae);
}

6
libnetdata/circular_buffer/circular_buffer.c
View File

@ -46,6 +46,11 @@ static int cbuffer_realloc_unsafe(struct circular_buffer *buf) {
return 0;
}
size_t cbuffer_available_size_unsafe(struct circular_buffer *buf) {
size_t len = (buf->write >= buf->read) ? (buf->write - buf->read) : (buf->size - buf->read + buf->write);
return buf->max_size - len;
}
int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len) {
size_t len = (buf->write >= buf->read) ? (buf->write - buf->read) : (buf->size - buf->read + buf->write);
while (d_len + len >= buf->size) {
@ -78,6 +83,7 @@ void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num) {
size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start) {
if (start != NULL)
*start = buf->data + buf->read;
if (buf->read <= buf->write) {
return buf->write - buf->read; // Includes empty case
}

2
libnetdata/circular_buffer/circular_buffer.h
View File

@ -13,4 +13,6 @@ extern void cbuffer_free(struct circular_buffer *buf);
extern int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len);
extern void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num);
extern size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start);
extern size_t cbuffer_available_size_unsafe(struct circular_buffer *buf);
#endif

626
libnetdata/dictionary/dictionary.c
View File

@ -1,28 +1,17 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// NOT TO BE USED BY USERS
#define DICTIONARY_FLAG_EXCLUSIVE_ACCESS (1 << 29) // there is only one thread accessing the dictionary
#define DICTIONARY_FLAG_DESTROYED (1 << 30) // this dictionary has been destroyed
#define DICTIONARY_FLAG_DEFER_ALL_DELETIONS (1 << 31) // defer all deletions of items in the dictionary
#define DICTIONARY_FLAG_EXCLUSIVE_ACCESS (1 << 28) // there is only one thread accessing the dictionary
#define DICTIONARY_FLAG_DESTROYED (1 << 29) // this dictionary has been destroyed
#define DICTIONARY_FLAG_DEFER_ALL_DELETIONS (1 << 30) // defer all deletions of items in the dictionary
// our reserved flags that cannot be set by users
#define DICTIONARY_FLAGS_RESERVED (DICTIONARY_FLAG_EXCLUSIVE_ACCESS|DICTIONARY_FLAG_DESTROYED|DICTIONARY_FLAG_DEFER_ALL_DELETIONS)
typedef struct dictionary DICTIONARY;
#define DICTIONARY_INTERNALS
#include "../libnetdata.h"
#ifndef ENABLE_DBENGINE
#define DICTIONARY_WITH_AVL
#warning Compiling DICTIONARY with an AVL index
#else
#define DICTIONARY_WITH_JUDYHS
#endif
#ifdef DICTIONARY_WITH_JUDYHS
#include <Judy.h>
#endif
typedef enum name_value_flags {
NAME_VALUE_FLAG_NONE = 0,
@ -38,10 +27,6 @@ typedef enum name_value_flags {
*/
typedef struct name_value {
#ifdef DICTIONARY_WITH_AVL
avl_t avl_node;
#endif
#ifdef NETDATA_INTERNAL_CHECKS
DICTIONARY *dict;
#endif
@ -72,15 +57,7 @@ struct dictionary {
NAME_VALUE *first_item; // the double linked list base pointers
NAME_VALUE *last_item;
#ifdef DICTIONARY_WITH_AVL
avl_tree_type values_index;
NAME_VALUE *hash_base;
void *(*get_thread_static_name_value)(const char *name);
#endif
#ifdef DICTIONARY_WITH_JUDYHS
Pvoid_t JudyHSArray; // the hash table
#endif
netdata_rwlock_t rwlock; // the r/w lock when DICTIONARY_FLAG_SINGLE_THREADED is not set
@ -315,9 +292,9 @@ static inline size_t dictionary_lock_free(DICTIONARY *dict) {
}
static void dictionary_lock(DICTIONARY *dict, char rw) {
if(rw == 'u' || rw == 'U') return;
if(rw == DICTIONARY_LOCK_NONE || rw == 'U') return;
if(rw == 'r' || rw == 'R') {
if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') {
// read lock
__atomic_add_fetch(&dict->readers, 1, __ATOMIC_RELAXED);
}
@ -329,7 +306,7 @@ static void dictionary_lock(DICTIONARY *dict, char rw) {
if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))
return;
if(rw == 'r' || rw == 'R') {
if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') {
// read lock
netdata_rwlock_rdlock(&dict->rwlock);
@ -347,9 +324,9 @@ static void dictionary_lock(DICTIONARY *dict, char rw) {
}
static void dictionary_unlock(DICTIONARY *dict, char rw) {
if(rw == 'u' || rw == 'U') return;
if(rw == DICTIONARY_LOCK_NONE || rw == 'U') return;
if(rw == 'r' || rw == 'R') {
if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') {
// read unlock
__atomic_sub_fetch(&dict->readers, 1, __ATOMIC_RELAXED);
}
@ -480,68 +457,6 @@ static uint32_t reference_counter_release(DICTIONARY *dict, NAME_VALUE *nv, bool
// ----------------------------------------------------------------------------
// hash table
#ifdef DICTIONARY_WITH_AVL
static inline const char *namevalue_get_name(NAME_VALUE *nv);
static int name_value_compare(void* a, void* b) {
return strcmp(namevalue_get_name((NAME_VALUE *)a), namevalue_get_name((NAME_VALUE *)b));
}
static void *get_thread_static_name_value(const char *name) {
static __thread NAME_VALUE tmp = { 0 };
tmp.flags = NAME_VALUE_FLAG_NONE;
tmp.caller_name = (char *)name;
return &tmp;
}
static void hashtable_init_unsafe(DICTIONARY *dict) {
avl_init(&dict->values_index, name_value_compare);
dict->get_thread_static_name_value = get_thread_static_name_value;
}
static size_t hashtable_destroy_unsafe(DICTIONARY *dict) {
(void)dict;
return 0;
}
static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *nv) {
(void)name;
(void)name_len;
if(unlikely(avl_remove(&(dict->values_index), (avl_t *)(nv)) != (avl_t *)nv))
return 0;
return 1;
}
static inline NAME_VALUE *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
(void)name_len;
void *tmp = dict->get_thread_static_name_value(name);
return (NAME_VALUE *)avl_search(&(dict->values_index), (avl_t *)tmp);
}
static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
// AVL needs a NAME_VALUE to insert into the dictionary but we don't have it yet.
// So, the only thing we can do, is return an existing one if it is already there.
// Returning NULL will make the caller thing we added it, will allocate one
// and will call hashtable_inserted_name_value_unsafe(), at which we will do
// the actual indexing.
dict->hash_base = hashtable_get_unsafe(dict, name, name_len);
return &dict->hash_base;
}
static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, void *nv) {
// we have our new NAME_VALUE object.
// Let's index it.
if(unlikely(avl_insert(&((dict)->values_index), (avl_t *)(nv)) != (avl_t *)nv))
error("dictionary: INTERNAL ERROR: duplicate insertion to dictionary.");
}
#endif
#ifdef DICTIONARY_WITH_JUDYHS
static void hashtable_init_unsafe(DICTIONARY *dict) {
dict->JudyHSArray = NULL;
}
@ -562,7 +477,7 @@ static size_t hashtable_destroy_unsafe(DICTIONARY *dict) {
return (size_t)ret;
}
static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
static inline void **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: inserting item from the index without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file);
JError_t J_Error;
@ -579,7 +494,7 @@ static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char
// put anything needed at the value of the index.
// The pointer to pointer we return has to be used before
// any other operation that may change the index (insert/delete).
return (NAME_VALUE **)Rc;
return Rc;
}
static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *nv) {
@ -632,8 +547,6 @@ static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, void *
;
}
#endif // DICTIONARY_WITH_JUDYHS
// ----------------------------------------------------------------------------
// linked list management
@ -979,7 +892,7 @@ static NAME_VALUE *dictionary_set_name_value_unsafe(DICTIONARY *dict, const char
// But the caller has the option to do this on his/her own.
// So, let's do the fastest here and let the caller decide the flow of calls.
NAME_VALUE *nv, **pnv = hashtable_insert_unsafe(dict, name, name_len);
NAME_VALUE *nv, **pnv = (NAME_VALUE **)hashtable_insert_unsafe(dict, name, name_len);
if(likely(*pnv == 0)) {
// a new item added to the index
nv = *pnv = namevalue_create_unsafe(dict, name, name_len, value, value_len);
@ -1280,6 +1193,9 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
dfe->value = NULL;
}
if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
dictionary_unlock(dfe->dict, dfe->rw);
return dfe->value;
}
@ -1294,6 +1210,9 @@ void *dictionary_foreach_next(DICTFE *dfe) {
return NULL;
}
if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
dictionary_lock(dfe->dict, dfe->rw);
// the item we just did
NAME_VALUE *nv = (NAME_VALUE *)dfe->last_item;
@ -1320,6 +1239,9 @@ void *dictionary_foreach_next(DICTFE *dfe) {
dfe->value = NULL;
}
if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
dictionary_unlock(dfe->dict, dfe->rw);
return dfe->value;
}
@ -1338,7 +1260,9 @@ usec_t dictionary_foreach_done(DICTFE *dfe) {
if(likely(nv))
reference_counter_release(dfe->dict, nv, false);
dictionary_unlock(dfe->dict, dfe->rw);
if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT))
dictionary_unlock(dfe->dict, dfe->rw);
dfe->dict = NULL;
dfe->last_item = NULL;
dfe->name = NULL;
@ -1383,8 +1307,14 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const c
// while we are using it
reference_counter_acquire(dict, nv);
if(unlikely(rw == DICTIONARY_LOCK_REENTRANT))
dictionary_unlock(dict, rw);
int r = callback(namevalue_get_name(nv), nv->value, data);
if(unlikely(rw == DICTIONARY_LOCK_REENTRANT))
dictionary_lock(dict, rw);
// since we have a reference counter, this item cannot be deleted
// until we release the reference counter, so the pointers are there
nv_next = nv->next;
@ -1449,7 +1379,15 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(
nv = array[i];
if(likely(!(nv->flags & NAME_VALUE_FLAG_DELETED))) {
reference_counter_acquire(dict, nv);
if(unlikely(rw == DICTIONARY_LOCK_REENTRANT))
dictionary_unlock(dict, rw);
int r = callback(namevalue_get_name(nv), nv->value, data);
if(unlikely(rw == DICTIONARY_LOCK_REENTRANT))
dictionary_lock(dict, rw);
reference_counter_release(dict, nv, false);
if (r < 0) {
ret = r;
@ -1469,132 +1407,293 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(
// ----------------------------------------------------------------------------
// STRING implementation - dedup all STRINGs
typedef struct string_entry {
#ifdef DICTIONARY_WITH_AVL
avl_t avl_node;
#endif
uint32_t length; // the string length with the terminating '\0'
uint32_t refcount; // how many times this string is used
const char str[]; // the string itself
} STRING_ENTRY;
struct netdata_string {
uint32_t length; // the string length including the terminating '\0'
#ifdef DICTIONARY_WITH_AVL
static int string_entry_compare(void* a, void* b) {
return strcmp(((STRING_ENTRY *)a)->str, ((STRING_ENTRY *)b)->str);
}
int32_t refcount; // how many times this string is used
// We use a signed number to be able to detect duplicate frees of a string.
// If at any point this goes below zero, we have a duplicate free.
static void *get_thread_static_string_entry(const char *name) {
static __thread size_t _length = 0;
static __thread STRING_ENTRY *_tmp = NULL;
size_t size = sizeof(STRING_ENTRY) + strlen(name) + 1;
if(likely(_tmp && _length < size)) {
freez(_tmp);
_tmp = NULL;
_length = 0;
}
if(unlikely(!_tmp)) {
_tmp = callocz(1, size);
_length = size;
}
strcpy((char *)&_tmp->str[0], name);
return _tmp;
}
#endif
DICTIONARY string_dictionary = {
#ifdef DICTIONARY_WITH_AVL
.values_index = {
.root = NULL,
.compar = string_entry_compare
},
.get_thread_static_name_value = get_thread_static_string_entry,
#endif
.flags = DICTIONARY_FLAG_EXCLUSIVE_ACCESS,
.rwlock = NETDATA_RWLOCK_INITIALIZER
const char str[]; // the string itself, is appended to this structure
};
static netdata_mutex_t string_mutex = NETDATA_MUTEX_INITIALIZER;
static struct string_hashtable {
Pvoid_t JudyHSArray; // the Judy array - hashtable
netdata_rwlock_t rwlock; // the R/W lock to protect the Judy array
long int entries; // the number of entries in the index
long int active_references; // the number of active references alive
long int memory; // the memory used, without the JudyHS index
size_t inserts; // the number of successful inserts to the index
size_t deletes; // the number of successful deleted from the index
size_t searches; // the number of successful searches in the index
size_t duplications; // when a string is referenced
size_t releases; // when a string is unreferenced
#ifdef NETDATA_INTERNAL_CHECKS
// internal statistics
size_t found_deleted_on_search;
size_t found_available_on_search;
size_t found_deleted_on_insert;
size_t found_available_on_insert;
size_t spins;
#endif
} string_base = {
.JudyHSArray = NULL,
.rwlock = NETDATA_RWLOCK_INITIALIZER,
};
#ifdef NETDATA_INTERNAL_CHECKS
#define string_internal_stats_add(var, val) __atomic_add_fetch(&string_base.var, val, __ATOMIC_RELAXED)
#else
#define string_internal_stats_add(var, val) do {;} while(0)
#endif
#define string_stats_atomic_increment(var) __atomic_add_fetch(&string_base.var, 1, __ATOMIC_RELAXED)
#define string_stats_atomic_decrement(var) __atomic_sub_fetch(&string_base.var, 1, __ATOMIC_RELAXED)
void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) {
*inserts = string_base.inserts;
*deletes = string_base.deletes;
*searches = string_base.searches;
*entries = (size_t)string_base.entries;
*references = (size_t)string_base.active_references;
*memory = (size_t)string_base.memory;
*duplications = string_base.duplications;
*releases = string_base.releases;
}
#define string_entry_acquire(se) __atomic_add_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST);
#define string_entry_release(se) __atomic_sub_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST);
static inline bool string_entry_check_and_acquire(STRING *se) {
int32_t expected, desired, count = 0;
do {
count++;
expected = se->refcount;
if(expected <= 0) {
// We cannot use this.
// The reference counter reached value zero,
// so another thread is deleting this.
string_internal_stats_add(spins, count - 1);
return false;
}
desired = expected + 1;
}
while(!__atomic_compare_exchange_n(&se->refcount, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
string_internal_stats_add(spins, count - 1);
// statistics
// string_base.active_references is altered at the in string_strdupz() and string_freez()
string_stats_atomic_increment(duplications);
return true;
}
STRING *string_dup(STRING *string) {
if(unlikely(!string)) return NULL;
STRING_ENTRY *se = (STRING_ENTRY *)string;
netdata_mutex_lock(&string_mutex);
se->refcount++;
netdata_mutex_unlock(&string_mutex);
#ifdef NETDATA_INTERNAL_CHECKS
if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) <= 0))
fatal("STRING: tried to %s() a string that is freed (it has %d references).", __FUNCTION__, string->refcount);
#endif
string_entry_acquire(string);
// statistics
string_stats_atomic_increment(active_references);
string_stats_atomic_increment(duplications);
return string;
}
// Search the index and return an ACQUIRED string entry, or NULL
static inline STRING *string_index_search(const char *str, size_t length) {
if(unlikely(!string_base.JudyHSArray))
return NULL;
STRING *string;
// Find the string in the index
// With a read-lock so that multiple readers can use the index concurrently.
netdata_rwlock_rdlock(&string_base.rwlock);
Pvoid_t *Rc;
Rc = JudyHSGet(string_base.JudyHSArray, (void *)str, length);
if(likely(Rc)) {
// found in the hash table
string = *Rc;
if(string_entry_check_and_acquire(string)) {
// we can use this entry
string_internal_stats_add(found_available_on_search, 1);
}
else {
// this entry is about to be deleted by another thread
// do not touch it, let it go...
string = NULL;
string_internal_stats_add(found_deleted_on_search, 1);
}
}
else {
// not found in the hash table
string = NULL;
}
string_stats_atomic_increment(searches);
netdata_rwlock_unlock(&string_base.rwlock);
return string;
}
// Insert a string to the index and return an ACQUIRED string entry,
// or NULL if the call needs to be retried (a deleted entry with the same key is still in the index)
// The returned entry is ACQUIRED and it can either be:
// 1. a new item inserted, or
// 2. an item found in the index that is not currently deleted
static inline STRING *string_index_insert(const char *str, size_t length) {
STRING *string;
netdata_rwlock_wrlock(&string_base.rwlock);
STRING **ptr;
{
JError_t J_Error;
Pvoid_t *Rc = JudyHSIns(&string_base.JudyHSArray, (void *)str, length, &J_Error);
if (unlikely(Rc == PJERR)) {
fatal(
"STRING: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d",
str,
JU_ERRNO(&J_Error),
JU_ERRID(&J_Error));
}
ptr = (STRING **)Rc;
}
if (likely(*ptr == 0)) {
// a new item added to the index
size_t mem_size = sizeof(STRING) + length;
string = mallocz(mem_size);
strcpy((char *)string->str, str);
string->length = length;
string->refcount = 1;
*ptr = string;
string_base.inserts++;
string_base.entries++;
string_base.memory += (long)mem_size;
}
else {
// the item is already in the index
string = *ptr;
if(string_entry_check_and_acquire(string)) {
// we can use this entry
string_internal_stats_add(found_available_on_insert, 1);
}
else {
// this entry is about to be deleted by another thread
// do not touch it, let it go...
string = NULL;
string_internal_stats_add(found_deleted_on_insert, 1);
}
string_stats_atomic_increment(searches);
}
netdata_rwlock_unlock(&string_base.rwlock);
return string;
}
// delete an entry from the index
static inline void string_index_delete(STRING *string) {
netdata_rwlock_wrlock(&string_base.rwlock);
#ifdef NETDATA_INTERNAL_CHECKS
if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) != 0))
fatal("STRING: tried to delete a string at %s() that is already freed (it has %d references).", __FUNCTION__, string->refcount);
#endif
bool deleted = false;
if (likely(string_base.JudyHSArray)) {
JError_t J_Error;
int ret = JudyHSDel(&string_base.JudyHSArray, (void *)string->str, string->length, &J_Error);
if (unlikely(ret == JERR)) {
error(
"STRING: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d",
string->str,
JU_ERRNO(&J_Error),
JU_ERRID(&J_Error));
} else
deleted = true;
}
if (unlikely(!deleted))
error("STRING: tried to delete '%s' that is not in the index. Ignoring it.", string->str);
else {
size_t mem_size = sizeof(STRING) + string->length;
string_base.deletes++;
string_base.entries--;
string_base.memory -= (long)mem_size;
freez(string);
}
netdata_rwlock_unlock(&string_base.rwlock);
}
STRING *string_strdupz(const char *str) {
if(unlikely(!str || !*str)) return NULL;
netdata_mutex_lock(&string_mutex);
size_t length = strlen(str) + 1;
STRING_ENTRY *se;
STRING_ENTRY **ptr = (STRING_ENTRY **)hashtable_insert_unsafe(&string_dictionary, str, length);
if(unlikely(*ptr == 0)) {
// a new item added to the index
size_t mem_size = sizeof(STRING_ENTRY) + length;
se = mallocz(mem_size);
strcpy((char *)se->str, str);
se->length = length;
se->refcount = 1;
*ptr = se;
hashtable_inserted_name_value_unsafe(&string_dictionary, se);
string_dictionary.version++;
string_dictionary.inserts++;
string_dictionary.entries++;
string_dictionary.memory += (long)mem_size;
}
else {
// the item is already in the index
se = *ptr;
se->refcount++;
string_dictionary.searches++;
STRING *string = string_index_search(str, length);
while(!string) {
// The search above did not find anything,
// We loop here, because during insert we may find an entry that is being deleted by another thread.
// So, we have to let it go and retry to insert it again.
string = string_index_insert(str, length);
}
netdata_mutex_unlock(&string_mutex);
return (STRING *)se;
// statistics
string_stats_atomic_increment(active_references);
return string;
}
void string_freez(STRING *string) {
if(unlikely(!string)) return;
netdata_mutex_lock(&string_mutex);
STRING_ENTRY *se = (STRING_ENTRY *)string;
int32_t refcount = string_entry_release(string);
if(se->refcount == 0)
fatal("STRING: tried to free string that has zero references.");
#ifdef NETDATA_INTERNAL_CHECKS
if(unlikely(refcount < 0))
fatal("STRING: tried to %s() a string that is already freed (it has %d references).", __FUNCTION__, string->refcount);
#endif
se->refcount--;
if(unlikely(se->refcount == 0)) {
if(hashtable_delete_unsafe(&string_dictionary, se->str, se->length, se) == 0)
error("STRING: INTERNAL ERROR: tried to delete '%s' that is not in the index", se->str);
if(unlikely(refcount == 0))
string_index_delete(string);
size_t mem_size = sizeof(STRING_ENTRY) + se->length;
freez(se);
string_dictionary.version++;
string_dictionary.deletes++;
string_dictionary.entries--;
string_dictionary.memory -= (long)mem_size;
}
netdata_mutex_unlock(&string_mutex);
// statistics
string_stats_atomic_decrement(active_references);
string_stats_atomic_increment(releases);
}
size_t string_length(STRING *string) {
size_t string_strlen(STRING *string) {
if(unlikely(!string)) return 0;
return ((STRING_ENTRY *)string)->length - 1;
return string->length - 1;
}
const char *string2str(STRING *string) {
if(unlikely(!string)) return "";
return ((STRING_ENTRY *)string)->str;
return string->str;
}
STRING *string_2way_merge(STRING *a, STRING *b) {
@ -1610,9 +1709,9 @@ STRING *string_2way_merge(STRING *a, STRING *b) {
if(unlikely(!a)) return string_dup(X);
if(unlikely(!b)) return string_dup(X);
size_t alen = string_length(a);
size_t blen = string_length(b);
size_t length = alen + blen + string_length(X) + 1;
size_t alen = string_strlen(a);
size_t blen = string_strlen(b);
size_t length = alen + blen + string_strlen(X) + 1;
char buf1[length + 1], buf2[length + 1], *dst1;
const char *s1, *s2;
@ -1642,6 +1741,52 @@ STRING *string_2way_merge(STRING *a, STRING *b) {
return string_strdupz(buf1);
}
// ----------------------------------------------------------------------------
// THREAD_CACHE
static __thread Pvoid_t thread_cache_judy_array = NULL;
void *thread_cache_entry_get_or_set(void *key,
ssize_t key_length,
void *value,
void *(*transform_the_value_before_insert)(void *key, size_t key_length, void *value)
) {
if(unlikely(!key || !key_length)) return NULL;
if(key_length == -1)
key_length = (ssize_t)strlen((char *)key) + 1;
JError_t J_Error;
Pvoid_t *Rc = JudyHSIns(&thread_cache_judy_array, key, key_length, &J_Error);
if (unlikely(Rc == PJERR)) {
fatal("THREAD_CACHE: Cannot insert entry to JudyHS, JU_ERRNO_* == %u, ID == %d",
JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
}
if(*Rc == 0) {
// new item added
*Rc = (transform_the_value_before_insert) ? transform_the_value_before_insert(key, key_length, value) : value;
}
return *Rc;
}
void thread_cache_destroy(void) {
if(unlikely(!thread_cache_judy_array)) return;
JError_t J_Error;
Word_t ret = JudyHSFreeArray(&thread_cache_judy_array, &J_Error);
if(unlikely(ret == (Word_t) JERR)) {
error("THREAD_CACHE: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d",
JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
}
internal_error(true, "THREAD_CACHE: hash table freed %lu bytes", ret);
thread_cache_judy_array = NULL;
}
// ----------------------------------------------------------------------------
// unit test
@ -2153,6 +2298,27 @@ static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, co
return errors;
}
static int string_threads_join = 0;
static void *string_thread(void *arg __maybe_unused) {
int dups = 1; //(gettid() % 10);
for(; 1 ;) {
if(string_threads_join)
break;
STRING *s = string_strdupz("string thread checking 1234567890");
for(int i = 0; i < dups ; i++)
string_dup(s);
for(int i = 0; i < dups ; i++)
string_freez(s);
string_freez(s);
}
return arg;
}
int dictionary_unittest(size_t entries) {
if(entries < 10) entries = 10;
@ -2291,7 +2457,7 @@ int dictionary_unittest(size_t entries) {
// check string
{
long string_entries_starting = dictionary_stats_entries(&string_dictionary);
long int string_entries_starting = string_base.entries;
fprintf(stderr, "\nChecking strings...\n");
@ -2312,8 +2478,7 @@ int dictionary_unittest(size_t entries) {
else
fprintf(stderr, "OK: cloning string are deduplicated\n");
STRING_ENTRY *se = (STRING_ENTRY *)s1;
if(se->refcount != 3) {
if(s1->refcount != 3) {
errors++;
fprintf(stderr, "ERROR: string refcount is not 3\n");
}
@ -2355,9 +2520,9 @@ int dictionary_unittest(size_t entries) {
freez(strings);
if(dictionary_stats_entries(&string_dictionary) != string_entries_starting + 2) {
if(string_base.entries != string_entries_starting + 2) {
errors++;
fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, dictionary_stats_entries(&string_dictionary));
fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, string_base.entries);
}
else
fprintf(stderr, "OK: strings dictionary has 2 items\n");
@ -2405,6 +2570,65 @@ int dictionary_unittest(size_t entries) {
dictionary_unittest_free_char_pp(names, entries);
dictionary_unittest_free_char_pp(values, entries);
{
#ifdef NETDATA_INTERNAL_CHECKS
size_t ofound_deleted_on_search = string_base.found_deleted_on_search,
ofound_available_on_search = string_base.found_available_on_search,
ofound_deleted_on_insert = string_base.found_deleted_on_insert,
ofound_available_on_insert = string_base.found_available_on_insert,
ospins = string_base.spins;
#endif
size_t oinserts, odeletes, osearches, oentries, oreferences, omemory, oduplications, oreleases;
string_statistics(&oinserts, &odeletes, &osearches, &oentries, &oreferences, &omemory, &oduplications, &oreleases);
time_t seconds_to_run = 5;
int threads_to_create = 2;
fprintf(
stderr,
"Checking string concurrency with %d threads for %ld seconds...\n",
threads_to_create,
seconds_to_run);
// check string concurrency
netdata_thread_t threads[threads_to_create];
string_threads_join = 0;
for (int i = 0; i < threads_to_create; i++) {
char buf[100 + 1];
snprintf(buf, 100, "string%d", i);
netdata_thread_create(
&threads[i], buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, string_thread, NULL);
}
sleep_usec(seconds_to_run * USEC_PER_SEC);
string_threads_join = 1;
for (int i = 0; i < threads_to_create; i++) {
void *retval;
netdata_thread_join(threads[i], &retval);
}
size_t inserts, deletes, searches, sentries, references, memory, duplications, releases;
string_statistics(&inserts, &deletes, &searches, &sentries, &references, &memory, &duplications, &releases);
fprintf(stderr, "inserts %zu, deletes %zu, searches %zu, entries %zu, references %zu, memory %zu, duplications %zu, releases %zu\n",
inserts - oinserts, deletes - odeletes, searches - osearches, sentries - oentries, references - oreferences, memory - omemory, duplications - oduplications, releases - oreleases);
#ifdef NETDATA_INTERNAL_CHECKS
size_t found_deleted_on_search = string_base.found_deleted_on_search,
found_available_on_search = string_base.found_available_on_search,
found_deleted_on_insert = string_base.found_deleted_on_insert,
found_available_on_insert = string_base.found_available_on_insert,
spins = string_base.spins;
fprintf(stderr, "on insert: %zu ok + %zu deleted\non search: %zu ok + %zu deleted\nspins: %zu\n",
found_available_on_insert - ofound_available_on_insert,
found_deleted_on_insert - ofound_deleted_on_insert,
found_available_on_search - ofound_available_on_search,
found_deleted_on_search - ofound_deleted_on_search,
spins - ospins
);
#endif
}
fprintf(stderr, "\n%zu errors found\n", errors);
return errors ? 1 : 0;
}

33
libnetdata/dictionary/dictionary.h
View File

@ -47,9 +47,9 @@ typedef enum dictionary_flags {
DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end)
// to change the value of the following, you also need to change the corresponding #defines in dictionary.c
DICTIONARY_FLAG_RESERVED1 = (1 << 29), // reserved for DICTIONARY_FLAG_EXCLUSIVE_ACCESS
DICTIONARY_FLAG_RESERVED2 = (1 << 30), // reserved for DICTIONARY_FLAG_DESTROYED
DICTIONARY_FLAG_RESERVED3 = (1 << 31), // reserved for DICTIONARY_FLAG_DEFER_ALL_DELETIONS
DICTIONARY_FLAG_RESERVED1 = (1 << 28), // reserved for DICTIONARY_FLAG_EXCLUSIVE_ACCESS
DICTIONARY_FLAG_RESERVED2 = (1 << 29), // reserved for DICTIONARY_FLAG_DESTROYED
DICTIONARY_FLAG_RESERVED3 = (1 << 30), // reserved for DICTIONARY_FLAG_DEFER_ALL_DELETIONS
} DICTIONARY_FLAGS;
// Create a dictionary
@ -179,9 +179,10 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(
#define DICTFE_CONST const
#endif
#define DICTIONARY_LOCK_READ 'r'
#define DICTIONARY_LOCK_WRITE 'w'
#define DICTIONARY_LOCK_NONE 'u'
#define DICTIONARY_LOCK_READ 'r'
#define DICTIONARY_LOCK_WRITE 'w'
#define DICTIONARY_LOCK_REENTRANT 'z'
#define DICTIONARY_LOCK_NONE 'u'
typedef DICTFE_CONST struct dictionary_foreach {
DICTFE_CONST char *name; // the dictionary name of the last item used
@ -235,7 +236,7 @@ typedef struct netdata_string STRING;
extern STRING *string_strdupz(const char *str);
extern STRING *string_dup(STRING *string);
extern void string_freez(STRING *string);
extern size_t string_length(STRING *string);
extern size_t string_strlen(STRING *string);
extern const char *string2str(STRING *string) NEVERNULL;
// keep common prefix/suffix and replace everything else with [x]
@ -243,10 +244,20 @@ extern STRING *string_2way_merge(STRING *a, STRING *b);
static inline int string_cmp(STRING *s1, STRING *s2) {
// STRINGs are deduplicated, so the same strings have the same pointer
if(unlikely(s1 == s2)) return 0;
// they differ, do the typical comparison
return strcmp(string2str(s1), string2str(s2));
// when they differ, we do the typical strcmp() comparison
return (s1 == s2)?0:strcmp(string2str(s1), string2str(s2));
}
extern void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases);
// ----------------------------------------------------------------------------
// THREAD CACHE
extern void *thread_cache_entry_get_or_set(void *key,
ssize_t key_length,
void *value,
void *(*transform_the_value_before_insert)(void *key, size_t key_length, void *value));
extern void thread_cache_destroy(void);
#endif /* NETDATA_DICTIONARY_H */

75
libnetdata/eval/eval.c
View File

@ -62,24 +62,36 @@ static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n);
// evaluation of expressions
static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) {
static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0;
static STRING
*this_string = NULL,
*now_string = NULL,
*after_string = NULL,
*before_string = NULL,
*status_string = NULL,
*removed_string = NULL,
*uninitialized_string = NULL,
*undefined_string = NULL,
*clear_string = NULL,
*warning_string = NULL,
*critical_string = NULL;
NETDATA_DOUBLE n;
if(unlikely(this_hash == 0)) {
this_hash = simple_hash("this");
now_hash = simple_hash("now");
after_hash = simple_hash("after");
before_hash = simple_hash("before");
status_hash = simple_hash("status");
removed_hash = simple_hash("REMOVED");
uninitialized_hash = simple_hash("UNINITIALIZED");
undefined_hash = simple_hash("UNDEFINED");
clear_hash = simple_hash("CLEAR");
warning_hash = simple_hash("WARNING");
critical_hash = simple_hash("CRITICAL");
if(unlikely(this_string == NULL)) {
this_string = string_strdupz("this");
now_string = string_strdupz("now");
after_string = string_strdupz("after");
before_string = string_strdupz("before");
status_string = string_strdupz("status");
removed_string = string_strdupz("REMOVED");
uninitialized_string = string_strdupz("UNINITIALIZED");
undefined_string = string_strdupz("UNDEFINED");
clear_string = string_strdupz("CLEAR");
warning_string = string_strdupz("WARNING");
critical_string = string_strdupz("CRITICAL");
}
if(unlikely(v->hash == this_hash && !strcmp(v->name, "this"))) {
if(unlikely(v->name == this_string)) {
n = (exp->myself)?*exp->myself:NAN;
buffer_strcat(exp->error_msg, "[ $this = ");
print_parsed_as_constant(exp->error_msg, n);
@ -87,7 +99,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == after_hash && !strcmp(v->name, "after"))) {
if(unlikely(v->name == after_string)) {
n = (exp->after && *exp->after)?*exp->after:NAN;
buffer_strcat(exp->error_msg, "[ $after = ");
print_parsed_as_constant(exp->error_msg, n);
@ -95,7 +107,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == before_hash && !strcmp(v->name, "before"))) {
if(unlikely(v->name == before_string)) {
n = (exp->before && *exp->before)?*exp->before:NAN;
buffer_strcat(exp->error_msg, "[ $before = ");
print_parsed_as_constant(exp->error_msg, n);
@ -103,15 +115,15 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == now_hash && !strcmp(v->name, "now"))) {
n = now_realtime_sec();
if(unlikely(v->name == now_string)) {
n = (NETDATA_DOUBLE)now_realtime_sec();
buffer_strcat(exp->error_msg, "[ $now = ");
print_parsed_as_constant(exp->error_msg, n);
buffer_strcat(exp->error_msg, " ] ");
return n;
}
if(unlikely(v->hash == status_hash && !strcmp(v->name, "status"))) {
if(unlikely(v->name == status_string)) {
n = (exp->status)?*exp->status:RRDCALC_STATUS_UNINITIALIZED;
buffer_strcat(exp->error_msg, "[ $status = ");
print_parsed_as_constant(exp->error_msg, n);
@ -119,7 +131,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == removed_hash && !strcmp(v->name, "REMOVED"))) {
if(unlikely(v->name == removed_string)) {
n = RRDCALC_STATUS_REMOVED;
buffer_strcat(exp->error_msg, "[ $REMOVED = ");
print_parsed_as_constant(exp->error_msg, n);
@ -127,7 +139,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == uninitialized_hash && !strcmp(v->name, "UNINITIALIZED"))) {
if(unlikely(v->name == uninitialized_string)) {
n = RRDCALC_STATUS_UNINITIALIZED;
buffer_strcat(exp->error_msg, "[ $UNINITIALIZED = ");
print_parsed_as_constant(exp->error_msg, n);
@ -135,7 +147,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == undefined_hash && !strcmp(v->name, "UNDEFINED"))) {
if(unlikely(v->name == undefined_string)) {
n = RRDCALC_STATUS_UNDEFINED;
buffer_strcat(exp->error_msg, "[ $UNDEFINED = ");
print_parsed_as_constant(exp->error_msg, n);
@ -143,7 +155,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == clear_hash && !strcmp(v->name, "CLEAR"))) {
if(unlikely(v->name == clear_string)) {
n = RRDCALC_STATUS_CLEAR;
buffer_strcat(exp->error_msg, "[ $CLEAR = ");
print_parsed_as_constant(exp->error_msg, n);
@ -151,7 +163,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == warning_hash && !strcmp(v->name, "WARNING"))) {
if(unlikely(v->name == warning_string)) {
n = RRDCALC_STATUS_WARNING;
buffer_strcat(exp->error_msg, "[ $WARNING = ");
print_parsed_as_constant(exp->error_msg, n);
@ -159,7 +171,7 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(unlikely(v->hash == critical_hash && !strcmp(v->name, "CRITICAL"))) {
if(unlikely(v->name == critical_string)) {
n = RRDCALC_STATUS_CRITICAL;
buffer_strcat(exp->error_msg, "[ $CRITICAL = ");
print_parsed_as_constant(exp->error_msg, n);
@ -167,15 +179,15 @@ static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *
return n;
}
if(exp->rrdcalc && health_variable_lookup(v->name, v->hash, exp->rrdcalc, &n)) {
buffer_sprintf(exp->error_msg, "[ ${%s} = ", v->name);
if(exp->rrdcalc && health_variable_lookup(v->name, exp->rrdcalc, &n)) {
buffer_sprintf(exp->error_msg, "[ ${%s} = ", string2str(v->name));
print_parsed_as_constant(exp->error_msg, n);
buffer_strcat(exp->error_msg, " ] ");
return n;
}
*error = EVAL_ERROR_UNKNOWN_VARIABLE;
buffer_sprintf(exp->error_msg, "[ undefined variable '%s' ] ", v->name);
buffer_sprintf(exp->error_msg, "[ undefined variable '%s' ] ", string2str(v->name));
return NAN;
}
@ -357,7 +369,7 @@ static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int
static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *error) {
(void)error;
buffer_sprintf(out, "${%s}", v->name);
buffer_sprintf(out, "${%s}", string2str(v->name));
}
static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n) {
@ -859,12 +871,11 @@ static inline void eval_node_set_value_to_variable(EVAL_NODE *op, int pos, const
op->ops[pos].type = EVAL_VALUE_VARIABLE;
op->ops[pos].variable = callocz(1, sizeof(EVAL_VARIABLE));
op->ops[pos].variable->name = strdupz(variable);
op->ops[pos].variable->hash = simple_hash(op->ops[pos].variable->name);
op->ops[pos].variable->name = string_strdupz(variable);
}
static inline void eval_variable_free(EVAL_VARIABLE *v) {
freez(v->name);
string_freez(v->name);
freez(v);
}

5
libnetdata/eval/eval.h
View File

@ -18,8 +18,7 @@ typedef enum rrdcalc_status {
} RRDCALC_STATUS;
typedef struct eval_variable {
char *name;
uint32_t hash;
STRING *name;
struct eval_variable *next;
} EVAL_VARIABLE;
@ -83,6 +82,6 @@ extern const char *expression_strerror(int error);
// 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg)
extern int expression_evaluate(EVAL_EXPRESSION *expression);
extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result);
extern int health_variable_lookup(STRING *variable, struct rrdcalc *rc, NETDATA_DOUBLE *result);
#endif //NETDATA_EVAL_H

70
libnetdata/libnetdata.h
View File

@ -214,6 +214,76 @@ extern "C" {
#define GUID_LEN 36
// ---------------------------------------------------------------------------------------------
// double linked list management
#define DOUBLE_LINKED_LIST_PREPEND_UNSAFE(head, item, prev, next) \
do { \
(item)->next = (head); \
\
if(likely(head)) { \
(item)->prev = (head)->prev; \
(head)->prev = (item); \
} \
else \
(item)->prev = (item); \
\
(head) = (item); \
} while (0)
#define DOUBLE_LINKED_LIST_APPEND_UNSAFE(head, item, prev, next) \
do { \
if(likely(head)) { \
(item)->prev = (head)->prev; \
(head)->prev->next = (item); \
(head)->prev = (item); \
(item)->next = NULL; \
} \
else { \
(head) = (item); \
(head)->prev = (head); \
(head)->next = NULL; \
} \
\
} while (0)
#define DOUBLE_LINKED_LIST_REMOVE_UNSAFE(head, item, prev, next) \
do { \
fatal_assert((head) != NULL); \
fatal_assert((item)->prev != NULL); \
\
if((item)->prev == (item)) { \
/* it is the only item in the list */ \
(head) = NULL; \
} \
else if((item) == (head)) { \
/* it is the first item */ \
(item)->next->prev = (item)->prev; \
(head) = (item)->next; \
} \
else { \
(item)->prev->next = (item)->next; \
if ((item)->next) { \
(item)->next->prev = (item)->prev; \
} \
else { \
(head)->prev = (item)->prev; \
} \
} \
\
(item)->next = NULL; \
(item)->prev = NULL; \
} while (0)
#define DOUBLE_LINKED_LIST_FOREACH_FORWARD(head, var, prev, next) \
for ((var) = (head); (var) ; (var) = (var)->next)
#define DOUBLE_LINKED_LIST_FOREACH_BACKWARD(head, var, prev, next) \
for ((var) = (head)?(head)->prev:NULL; (var) && (var) != (head)->prev ; (var) = (var)->prev)
// ---------------------------------------------------------------------------------------------
extern void netdata_fix_chart_id(char *s);
extern void netdata_fix_chart_name(char *s);

4
libnetdata/log/log.c
View File

@ -747,7 +747,7 @@ void debug_int( const char *file, const char *function, const unsigned long line
// ----------------------------------------------------------------------------
// info log
void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... )
void info_int( const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... )
{
va_list args;
@ -807,7 +807,7 @@ static const char *strerror_result_string(const char *a, const char *b) { (void)
#error "cannot detect the format of function strerror_r()"
#endif
void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
void error_int( const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
// save a copy of errno - just in case this function generates a new error
int __errno = errno;

5
libnetdata/required_dummies.h
View File

@ -22,15 +22,16 @@ void signals_block(void){};
void signals_unblock(void){};
void signals_reset(void){};
#ifndef UNIT_TESTING
// callback required by eval()
int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result)
int health_variable_lookup(STRING *variable, struct rrdcalc *rc, NETDATA_DOUBLE *result)
{
(void)variable;
(void)hash;
(void)rc;
(void)result;
return 0;
};
#endif
// required by get_system_cpus()
char *netdata_configured_host_prefix = "";

4
libnetdata/simple_pattern/simple_pattern.c
View File

@ -333,9 +333,7 @@ extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p)
}
char *simple_pattern_trim_around_equal(char *src) {
char *store = mallocz(strlen(src) +1);
if(!store)
return NULL;
char *store = mallocz(strlen(src) + 1);
char *dst = store;
while (*src) {

2
libnetdata/threads/threads.c
View File

@ -106,6 +106,8 @@ static void thread_cleanup(void *ptr) {
if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP))
info("thread with task id %d finished", gettid());
thread_cache_destroy();
freez((void *)netdata_thread->tag);
netdata_thread->tag = NULL;

145
libnetdata/worker_utilization/worker_utilization.c
View File

@ -4,15 +4,20 @@
#define WORKER_BUSY 'B'
struct worker_job_type {
char name[WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH + 1];
STRING *name;
STRING *units;
// statistics controlled variables
size_t statistics_last_jobs_started;
usec_t statistics_last_busy_time;
NETDATA_DOUBLE statistics_last_custom_value;
// worker controlled variables
volatile size_t worker_jobs_started;
volatile usec_t worker_busy_time;
WORKER_METRIC_TYPE type;
NETDATA_DOUBLE custom_value;
};
struct worker {
@ -36,6 +41,7 @@ struct worker {
struct worker_job_type per_job_type[WORKER_UTILIZATION_MAX_JOB_TYPES];
struct worker *next;
struct worker *prev;
};
static netdata_mutex_t base_lock = NETDATA_MUTEX_INITIALIZER;
@ -57,40 +63,44 @@ void worker_register(const char *workname) {
worker->last_action = WORKER_IDLE;
netdata_mutex_lock(&base_lock);
worker->next = base;
base = worker;
DOUBLE_LINKED_LIST_PREPEND_UNSAFE(base, worker, prev, next);
netdata_mutex_unlock(&base_lock);
}
void worker_register_job_name(size_t job_id, const char *name) {
void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type) {
if(unlikely(!worker)) return;
if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES)) {
error("WORKER_UTILIZATION: job_id %zu is too big. Max is %zu", job_id, (size_t)(WORKER_UTILIZATION_MAX_JOB_TYPES - 1));
return;
}
if (*worker->per_job_type[job_id].name) {
error("WORKER_UTILIZATION: duplicate job registration: worker '%s' job id %zu is '%s', ignoring '%s'", worker->workname, job_id, worker->per_job_type[job_id].name, name);
if(worker->per_job_type[job_id].name) {
if(strcmp(string2str(worker->per_job_type[job_id].name), name) != 0 || worker->per_job_type[job_id].type != type || strcmp(string2str(worker->per_job_type[job_id].units), units) != 0)
error("WORKER_UTILIZATION: duplicate job registration: worker '%s' job id %zu is '%s', ignoring the later '%s'", worker->workname, job_id, string2str(worker->per_job_type[job_id].name), name);
return;
}
strncpy(worker->per_job_type[job_id].name, name, WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH);
worker->per_job_type[job_id].name = string_strdupz(name);
worker->per_job_type[job_id].units = string_strdupz(units);
worker->per_job_type[job_id].type = type;
}
void worker_register_job_name(size_t job_id, const char *name) {
worker_register_job_custom_metric(job_id, name, "", WORKER_METRIC_IDLE_BUSY);
}
void worker_unregister(void) {
if(unlikely(!worker)) return;
netdata_mutex_lock(&base_lock);
if(base == worker)
base = worker->next;
else {
struct worker *p;
for(p = base; p && p->next && p->next != worker ;p = p->next);
if(p && p->next == worker)
p->next = worker->next;
}
DOUBLE_LINKED_LIST_REMOVE_UNSAFE(base, worker, prev, next);
netdata_mutex_unlock(&base_lock);
for(int i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
string_freez(worker->per_job_type[i].name);
string_freez(worker->per_job_type[i].units);
}
freez((void *)worker->tag);
freez((void *)worker->workname);
freez(worker);
@ -112,16 +122,14 @@ static inline void worker_is_idle_with_time(usec_t now) {
}
void worker_is_idle(void) {
if(unlikely(!worker)) return;
if(unlikely(worker->last_action != WORKER_BUSY)) return;
if(unlikely(!worker || worker->last_action != WORKER_BUSY)) return;
worker_is_idle_with_time(now_realtime_usec());
}
void worker_is_busy(size_t job_id) {
if(unlikely(!worker)) return;
if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))
job_id = 0;
if(unlikely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))
return;
usec_t now = now_realtime_usec();
@ -138,35 +146,97 @@ void worker_is_busy(size_t job_id) {
worker->last_action = WORKER_BUSY;
}
void worker_set_metric(size_t job_id, NETDATA_DOUBLE value) {
if(unlikely(!worker)) return;
if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))
return;
if(worker->per_job_type[job_id].type == WORKER_METRIC_INCREMENTAL)
worker->per_job_type[job_id].custom_value += value;
else
worker->per_job_type[job_id].custom_value = value;
}
// statistics interface
void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pid, const char *thread_tag, size_t utilization_usec, size_t duration_usec, size_t jobs_started, size_t is_running, const char **job_types_names, size_t *job_types_jobs_started, usec_t *job_types_busy_time), void *data) {
void workers_foreach(const char *workname, void (*callback)(
void *data
, pid_t pid
, const char *thread_tag
, size_t utilization_usec
, size_t duration_usec
, size_t jobs_started, size_t is_running
, STRING **job_types_names
, STRING **job_types_units
, WORKER_METRIC_TYPE *job_metric_types
, size_t *job_types_jobs_started
, usec_t *job_types_busy_time
, NETDATA_DOUBLE *job_custom_values
)
, void *data) {
netdata_mutex_lock(&base_lock);
uint32_t hash = simple_hash(workname);
usec_t busy_time, delta;
size_t i, jobs_started, jobs_running;
struct worker *p;
for(p = base; p ; p = p->next) {
if(hash != p->workname_hash || strcmp(workname, p->workname)) continue;
DOUBLE_LINKED_LIST_FOREACH_FORWARD(base, p, prev, next) {
if(hash != p->workname_hash || strcmp(workname, p->workname) != 0) continue;
usec_t now = now_realtime_usec();
// find per job type statistics
const char *per_job_type_name[WORKER_UTILIZATION_MAX_JOB_TYPES];
STRING *per_job_type_name[WORKER_UTILIZATION_MAX_JOB_TYPES];
STRING *per_job_type_units[WORKER_UTILIZATION_MAX_JOB_TYPES];
WORKER_METRIC_TYPE per_job_metric_type[WORKER_UTILIZATION_MAX_JOB_TYPES];
size_t per_job_type_jobs_started[WORKER_UTILIZATION_MAX_JOB_TYPES];
usec_t per_job_type_busy_time[WORKER_UTILIZATION_MAX_JOB_TYPES];
NETDATA_DOUBLE per_job_custom_values[WORKER_UTILIZATION_MAX_JOB_TYPES];
for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
per_job_type_name[i] = p->per_job_type[i].name;
per_job_type_units[i] = p->per_job_type[i].units;
per_job_metric_type[i] = p->per_job_type[i].type;
size_t tmp_jobs_started = p->per_job_type[i].worker_jobs_started;
per_job_type_jobs_started[i] = tmp_jobs_started - p->per_job_type[i].statistics_last_jobs_started;
p->per_job_type[i].statistics_last_jobs_started = tmp_jobs_started;
switch(p->per_job_type[i].type) {
default:
case WORKER_METRIC_EMPTY:
per_job_type_jobs_started[i] = 0;
per_job_type_busy_time[i] = 0;
per_job_custom_values[i] = NAN;
break;
usec_t tmp_busy_time = p->per_job_type[i].worker_busy_time;
per_job_type_busy_time[i] = tmp_busy_time - p->per_job_type[i].statistics_last_busy_time;
p->per_job_type[i].statistics_last_busy_time = tmp_busy_time;
case WORKER_METRIC_IDLE_BUSY: {
size_t tmp_jobs_started = p->per_job_type[i].worker_jobs_started;
per_job_type_jobs_started[i] = tmp_jobs_started - p->per_job_type[i].statistics_last_jobs_started;
p->per_job_type[i].statistics_last_jobs_started = tmp_jobs_started;
usec_t tmp_busy_time = p->per_job_type[i].worker_busy_time;
per_job_type_busy_time[i] = tmp_busy_time - p->per_job_type[i].statistics_last_busy_time;
p->per_job_type[i].statistics_last_busy_time = tmp_busy_time;
per_job_custom_values[i] = NAN;
break;
}
case WORKER_METRIC_ABSOLUTE:
per_job_type_jobs_started[i] = 0;
per_job_type_busy_time[i] = 0;
per_job_custom_values[i] = p->per_job_type[i].custom_value;
break;
case WORKER_METRIC_INCREMENTAL: {
per_job_type_jobs_started[i] = 0;
per_job_type_busy_time[i] = 0;
NETDATA_DOUBLE tmp_custom_value = p->per_job_type[i].custom_value;
per_job_custom_values[i] = tmp_custom_value - p->per_job_type[i].statistics_last_custom_value;
p->per_job_type[i].statistics_last_custom_value = tmp_custom_value;
break;
}
}
}
// get a copy of the worker variables
@ -203,7 +273,20 @@ void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pi
jobs_running = 1;
}
callback(data, p->pid, p->tag, busy_time, delta, jobs_started, jobs_running, per_job_type_name, per_job_type_jobs_started, per_job_type_busy_time);
callback(data
, p->pid
, p->tag
, busy_time
, delta
, jobs_started
, jobs_running
, per_job_type_name
, per_job_type_units
, per_job_metric_type
, per_job_type_jobs_started
, per_job_type_busy_time
, per_job_custom_values
);
}
netdata_mutex_unlock(&base_lock);

27
libnetdata/worker_utilization/worker_utilization.h
View File

@ -6,17 +6,40 @@
// workers interfaces
#define WORKER_UTILIZATION_MAX_JOB_TYPES 50
#define WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH 25
typedef enum {
WORKER_METRIC_EMPTY = 0,
WORKER_METRIC_IDLE_BUSY = 1,
WORKER_METRIC_ABSOLUTE = 2,
WORKER_METRIC_INCREMENTAL = 3,
} WORKER_METRIC_TYPE;
extern void worker_register(const char *workname);
extern void worker_register_job_name(size_t job_id, const char *name);
extern void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type);
extern void worker_unregister(void);
extern void worker_is_idle(void);
extern void worker_is_busy(size_t job_id);
extern void worker_set_metric(size_t job_id, NETDATA_DOUBLE value);
// statistics interface
extern void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pid, const char *thread_tag, size_t utilization_usec, size_t duration_usec, size_t jobs_started, size_t is_running, const char **job_types_names, size_t *job_types_jobs_started, usec_t *job_types_busy_time), void *data);
extern void workers_foreach(const char *workname, void (*callback)(
void *data
, pid_t pid
, const char *thread_tag
, size_t utilization_usec
, size_t duration_usec
, size_t jobs_started
, size_t is_running
, STRING **job_types_names
, STRING **job_types_units
, WORKER_METRIC_TYPE *job_metric_types
, size_t *job_types_jobs_started
, usec_t *job_types_busy_time
, NETDATA_DOUBLE *job_custom_values
)
, void *data);
#endif // WORKER_UTILIZATION_H

2
ml/Dimension.h
View File

@ -26,7 +26,7 @@ public:
RRDSET *RS = RD->rrdset;
std::stringstream SS;
SS << RS->context << "|" << RS->id << "|" << RD->name;
SS << rrdset_context(RS) << "|" << rrdset_id(RS) << "|" << rrddim_name(RD);
return SS.str();
}

12
ml/Host.cc
View File

@ -23,7 +23,7 @@ static void updateDimensionsChart(RRDHOST *RH,
std::stringstream IdSS, NameSS;
IdSS << "dimensions_on_" << localhost->machine_guid;
NameSS << "dimensions_on_" << localhost->hostname;
NameSS << "dimensions_on_" << rrdhost_hostname(localhost);
RS = rrdset_create(
RH,
@ -69,7 +69,7 @@ static void updateRateChart(RRDHOST *RH, collected_number AnomalyRate) {
std::stringstream IdSS, NameSS;
IdSS << "anomaly_rate_on_" << localhost->machine_guid;
NameSS << "anomaly_rate_on_" << localhost->hostname;
NameSS << "anomaly_rate_on_" << rrdhost_hostname(localhost);
RS = rrdset_create(
RH,
@ -106,7 +106,7 @@ static void updateWindowLengthChart(RRDHOST *RH, collected_number WindowLength)
std::stringstream IdSS, NameSS;
IdSS << "detector_window_on_" << localhost->machine_guid;
NameSS << "detector_window_on_" << localhost->hostname;
NameSS << "detector_window_on_" << rrdhost_hostname(localhost);
RS = rrdset_create(
RH,
@ -147,7 +147,7 @@ static void updateEventsChart(RRDHOST *RH,
std::stringstream IdSS, NameSS;
IdSS << "detector_events_on_" << localhost->machine_guid;
NameSS << "detector_events_on_" << localhost->hostname;
NameSS << "detector_events_on_" << rrdhost_hostname(localhost);
RS = rrdset_create(
RH,
@ -193,7 +193,7 @@ static void updateDetectionChart(RRDHOST *RH) {
std::stringstream IdSS, NameSS;
IdSS << "prediction_stats_" << RH->machine_guid;
NameSS << "prediction_stats_for_" << RH->hostname;
NameSS << "prediction_stats_for_" << rrdhost_hostname(RH);
RS = rrdset_create_localhost(
"netdata", // type
@ -233,7 +233,7 @@ static void updateTrainingChart(RRDHOST *RH, struct rusage *TRU)
std::stringstream IdSS, NameSS;
IdSS << "training_stats_" << RH->machine_guid;
NameSS << "training_stats_for_" << RH->hostname;
NameSS << "training_stats_for_" << rrdhost_hostname(RH);
RS = rrdset_create_localhost(
"netdata", // type

4
ml/Host.h
View File

@ -31,9 +31,7 @@ public:
RRDSET_TYPE_LINE
);
AnomalyRateRS->flags = static_cast<RRDSET_FLAGS>(
static_cast<int>(AnomalyRateRS->flags) | RRDSET_FLAG_HIDDEN
);
rrdset_flag_set(AnomalyRateRS, RRDSET_FLAG_HIDDEN);
}
RRDHOST *getRH() { return RH; }

4
ml/ml.cc
View File

@ -16,7 +16,7 @@ bool ml_enabled(RRDHOST *RH) {
if (!Cfg.EnableAnomalyDetection)
return false;
if (simple_pattern_matches(Cfg.SP_HostsToSkip, RH->hostname))
if (simple_pattern_matches(Cfg.SP_HostsToSkip, rrdhost_hostname(RH)))
return false;
return true;
@ -76,7 +76,7 @@ void ml_new_dimension(RRDDIM *RD) {
if (static_cast<unsigned>(RD->update_every) != H->updateEvery())
return;
if (simple_pattern_matches(Cfg.SP_ChartsToSkip, RS->name))
if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RS)))
return;
Dimension *D = new Dimension(RD);

5
parser/parser.c
View File

@ -38,6 +38,7 @@ PARSER *parser_init(RRDHOST *host, void *user, void *input, PARSER_INPUT_TYPE fl
parser->input = input;
parser->flags = flags;
parser->host = host;
parser->worker_job_next_id = WORKER_PARSER_FIRST_JOB;
#ifdef ENABLE_HTTPS
parser->bytesleft = 0;
@ -133,7 +134,7 @@ int parser_add_keyword(PARSER *parser, char *keyword, keyword_function func)
tmp_keyword = callocz(1, sizeof(*tmp_keyword));
tmp_keyword->worker_job_id = parser->worker_job_ids++;
tmp_keyword->worker_job_id = parser->worker_job_next_id++;
tmp_keyword->keyword = strdupz(keyword);
tmp_keyword->keyword_hash = keyword_hash;
tmp_keyword->func[tmp_keyword->func_no++] = (void *) func;
@ -268,7 +269,7 @@ inline int parser_action(PARSER *parser, char *input)
uint32_t command_hash = simple_hash(command);
size_t worker_job_id = 0;
size_t worker_job_id = WORKER_UTILIZATION_MAX_JOB_TYPES + 1; // set an invalid value by default
while(tmp_keyword) {
if (command_hash == tmp_keyword->keyword_hash &&
(!strcmp(command, tmp_keyword->keyword))) {

3
parser/parser.h
View File

@ -7,6 +7,7 @@
#define PARSER_MAX_CALLBACKS 20
#define PARSER_MAX_RECOVER_KEYWORDS 128
#define WORKER_PARSER_FIRST_JOB 1
// PARSER return codes
typedef enum parser_rc {
@ -68,7 +69,7 @@ typedef struct parser_data {
} PARSER_DATA;
typedef struct parser {
size_t worker_job_ids;
size_t worker_job_next_id;
uint8_t version; // Parser version
RRDHOST *host;
void *input; // Input source e.g. stream

2
registry/registry.c
View File

@ -61,7 +61,7 @@ static inline void registry_json_header(RRDHOST *host, struct web_client *w, con
buffer_flush(w->response.data);
w->response.data->contenttype = CT_APPLICATION_JSON;
buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"",
action, status, host->registry_hostname, host->machine_guid);
action, status, rrdhost_registry_hostname(host), host->machine_guid);
}
static inline void registry_json_footer(struct web_client *w) {

71
streaming/compression.c
View File

@ -5,9 +5,6 @@
#define STREAM_COMPRESSION_MSG "STREAM_COMPRESSION"
#define LZ4_MAX_MSG_SIZE 0x4000
#define LZ4_STREAM_BUFFER_SIZE (0x10000 + LZ4_MAX_MSG_SIZE)
#define SIGNATURE ((uint32_t)('z' | 0x80) | (0x80 << 8) | (0x80 << 16) | ('\n' << 24))
#define SIGNATURE_MASK ((uint32_t)0xff | (0x80 << 8) | (0x80 << 16) | (0xff << 24))
#define SIGNATURE_SIZE 4
@ -18,8 +15,9 @@
*/
struct compressor_data {
LZ4_stream_t *stream;
char *stream_buffer;
size_t stream_buffer_pos;
char *input_ring_buffer;
size_t input_ring_buffer_size;
size_t input_ring_buffer_pos;
};
@ -33,7 +31,7 @@ static void lz4_compressor_reset(struct compressor_state *state)
LZ4_resetStream_fast(state->data->stream);
info("%s: Compressor Reset", STREAM_COMPRESSION_MSG);
}
state->data->stream_buffer_pos = 0;
state->data->input_ring_buffer_pos = 0;
}
}
@ -47,10 +45,10 @@ static void lz4_compressor_destroy(struct compressor_state **state)
if (s->data) {
if (s->data->stream)
LZ4_freeStream(s->data->stream);
freez(s->data->stream_buffer);
freez(s->data->input_ring_buffer);
freez(s->data);
}
freez(s->buffer);
freez(s->compression_result_buffer);
freez(s);
*state = NULL;
debug(D_STREAM, "%s: Compressor Destroyed.", STREAM_COMPRESSION_MSG);
@ -65,37 +63,53 @@ static void lz4_compressor_destroy(struct compressor_state **state)
*/
static size_t lz4_compressor_compress(struct compressor_state *state, const char *data, size_t size, char **out)
{
if (!state || !size || !out)
if(unlikely(!state || !size || !out))
return 0;
if (size > LZ4_MAX_MSG_SIZE) {
if(unlikely(size > LZ4_MAX_MSG_SIZE)) {
error("%s: Compression Failed - Message size %lu above compression buffer limit: %d", STREAM_COMPRESSION_MSG, size, LZ4_MAX_MSG_SIZE);
return 0;
}
size_t max_dst_size = LZ4_COMPRESSBOUND(size);
size_t data_size = max_dst_size + SIGNATURE_SIZE;
if (!state->buffer) {
state->buffer = mallocz(data_size);
state->buffer_size = data_size;
} else if (state->buffer_size < data_size) {
state->buffer = reallocz(state->buffer, data_size);
state->buffer_size = data_size;
if (!state->compression_result_buffer) {
state->compression_result_buffer = mallocz(data_size);
state->compression_result_buffer_size = data_size;
}
else if(unlikely(state->compression_result_buffer_size < data_size)) {
state->compression_result_buffer = reallocz(state->compression_result_buffer, data_size);
state->compression_result_buffer_size = data_size;
}
memcpy(state->data->stream_buffer + state->data->stream_buffer_pos, data, size);
long int compressed_data_size = LZ4_compress_fast_continue(state->data->stream,
state->data->stream_buffer + state->data->stream_buffer_pos,
state->buffer + SIGNATURE_SIZE, size, max_dst_size, 1);
// the ring buffer always has space for LZ4_MAX_MSG_SIZE
memcpy(state->data->input_ring_buffer + state->data->input_ring_buffer_pos, data, size);
// this call needs the last 64K of our previous data
// they are available in the ring buffer
long int compressed_data_size = LZ4_compress_fast_continue(
state->data->stream,
state->data->input_ring_buffer + state->data->input_ring_buffer_pos,
state->compression_result_buffer + SIGNATURE_SIZE,
size,
max_dst_size,
1);
if (compressed_data_size < 0) {
error("Data compression error: %ld", compressed_data_size);
return 0;
}
state->data->stream_buffer_pos += size;
if (state->data->stream_buffer_pos >= LZ4_STREAM_BUFFER_SIZE - LZ4_MAX_MSG_SIZE)
state->data->stream_buffer_pos = 0;
// update the next writing position of the ring buffer
state->data->input_ring_buffer_pos += size;
if(unlikely(state->data->input_ring_buffer_pos >= state->data->input_ring_buffer_size - LZ4_MAX_MSG_SIZE))
state->data->input_ring_buffer_pos = 0;
// update the signature header
uint32_t len = ((compressed_data_size & 0x7f) | 0x80 | (((compressed_data_size & (0x7f << 7)) << 1) | 0x8000)) << 8;
*(uint32_t *)state->buffer = len | SIGNATURE;
*out = state->buffer;
*(uint32_t *)state->compression_result_buffer = len | SIGNATURE;
*out = state->compression_result_buffer;
debug(D_STREAM, "%s: Compressed data header: %ld", STREAM_COMPRESSION_MSG, compressed_data_size);
return compressed_data_size + SIGNATURE_SIZE;
}
@ -114,8 +128,9 @@ struct compressor_state *create_compressor()
state->data = callocz(1, sizeof(struct compressor_data));
state->data->stream = LZ4_createStream();
state->data->stream_buffer = callocz(1, LZ4_DECODER_RING_BUFFER_SIZE(LZ4_MAX_MSG_SIZE));
state->buffer_size = LZ4_STREAM_BUFFER_SIZE;
state->data->input_ring_buffer_size = LZ4_DECODER_RING_BUFFER_SIZE(LZ4_MAX_MSG_SIZE * 2);
state->data->input_ring_buffer = callocz(1, state->data->input_ring_buffer_size);
state->compression_result_buffer_size = 0;
state->reset(state);
debug(D_STREAM, "%s: Initialize streaming compression!", STREAM_COMPRESSION_MSG);
return state;
@ -281,6 +296,8 @@ static size_t lz4_decompressor_decompress(struct decompressor_state *state)
size_t avg_saving = saving_percent(state->total_compressed, state->total_uncompressed);
size_t avg_size = state->total_uncompressed / state->packet_count;
(void)saving;
if (old_avg_saving != avg_saving || old_avg_size != avg_size){
debug(D_STREAM, "%s: Saving: %lu%% (avg. %lu%%), avg.size: %lu", STREAM_COMPRESSION_MSG, saving, avg_saving, avg_size);
}

72
streaming/receiver.c
View File

@ -1,6 +1,13 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "rrdpush.h"
#include "parser/parser.h"
#define WORKER_RECEIVER_JOB_BYTES_READ (WORKER_PARSER_FIRST_JOB - 1)
#if WORKER_PARSER_FIRST_JOB < 1
#error The define WORKER_PARSER_FIRST_JOB needs to be at least 1
#endif
extern struct config stream_config;
@ -66,7 +73,7 @@ PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins
RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host;
struct plugind *cd = ((PARSER_USER_OBJECT *)user)->cd;
if (cd->version < VERSION_GAP_FILLING ) {
error("STREAM %s from %s: Child negotiated version %u but sent TIMESTAMP!", host->hostname, cd->cmd,
error("STREAM %s from %s: Child negotiated version %u but sent TIMESTAMP!", rrdhost_hostname(host), cd->cmd,
cd->version);
return PARSER_RC_OK; // Ignore error and continue stream
}
@ -78,7 +85,7 @@ PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins
info(
"STREAM %s from %s: Initial connection (no gap to check), "
"remote=%"PRId64" local=%"PRId64" slew=%"PRId64"",
host->hostname,
rrdhost_hostname(host),
cd->cmd,
(int64_t)remote_time,
(int64_t)now,
@ -88,7 +95,7 @@ PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins
info(
"STREAM %s from %s: Checking for gaps... "
"remote=%"PRId64" local=%"PRId64"..%"PRId64" slew=%"PRId64" %"PRId64"-sec gap",
host->hostname,
rrdhost_hostname(host),
cd->cmd,
(int64_t)remote_time,
(int64_t)prev,
@ -177,6 +184,7 @@ static int receiver_read(struct receiver_state *r, FILE *fp) {
int ret = SSL_read(r->ssl.conn, r->read_buffer + r->read_len, desired);
if (ret > 0 ) {
r->read_len += ret;
worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret);
return 0;
}
// Don't treat SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE differently on blocking socket
@ -192,6 +200,7 @@ static int receiver_read(struct receiver_state *r, FILE *fp) {
if (!fgets(r->read_buffer, sizeof(r->read_buffer), fp))
return 1;
r->read_len = strlen(r->read_buffer);
worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, r->read_len);
return 0;
}
#else
@ -279,22 +288,24 @@ static int read_stream(struct receiver_state *r, FILE *fp, char* buffer, size_t
*/
static int receiver_read(struct receiver_state *r, FILE *fp) {
// check any decompressed data present
if (r->decompressor &&
r->decompressor->decompressed_bytes_in_buffer(r->decompressor)) {
if (r->decompressor && r->decompressor->decompressed_bytes_in_buffer(r->decompressor)) {
size_t available = sizeof(r->read_buffer) - r->read_len;
if (available) {
size_t len = r->decompressor->get(r->decompressor,
r->read_buffer + r->read_len, available);
size_t len = r->decompressor->get(r->decompressor, r->read_buffer + r->read_len, available);
if (!len)
return 1;
r->read_len += len;
}
return 0;
}
int ret = 0;
if (read_stream(r, fp, r->read_buffer + r->read_len, sizeof(r->read_buffer) - r->read_len - 1, &ret))
return 1;
worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret);
if (!is_compressed_data(r->read_buffer, ret)) {
r->read_len += ret;
return 0;
@ -303,8 +314,7 @@ static int receiver_read(struct receiver_state *r, FILE *fp) {
if (unlikely(!r->decompressor))
r->decompressor = create_decompressor();
size_t bytes_to_read = r->decompressor->start(r->decompressor,
r->read_buffer, ret);
size_t bytes_to_read = r->decompressor->start(r->decompressor, r->read_buffer, ret);
// Read the entire block of compressed data because
// we're unable to decompress incomplete block
@ -312,18 +322,23 @@ static int receiver_read(struct receiver_state *r, FILE *fp) {
do {
if (read_stream(r, fp, compressed, bytes_to_read, &ret))
return 1;
worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret);
// Send input data to decompressor
if (ret)
r->decompressor->put(r->decompressor, compressed, ret);
bytes_to_read -= ret;
} while (bytes_to_read > 0);
// Decompress
size_t bytes_to_parse = r->decompressor->decompress(r->decompressor);
if (!bytes_to_parse)
return 1;
// Fill read buffer with decompressed data
r->read_len = r->decompressor->get(r->decompressor,
r->read_buffer, sizeof(r->read_buffer));
r->read_len = r->decompressor->get(r->decompressor, r->read_buffer, sizeof(r->read_buffer));
return 0;
}
@ -486,8 +501,8 @@ static int rrdpush_receive(struct receiver_state *rpt)
#else
if(send_timeout(rpt->fd, initial_response, strlen(initial_response), 0, 60) != strlen(initial_response)) {
#endif
log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "FAILED - CANNOT REPLY");
error("STREAM %s [receive from [%s]:%s]: cannot send command.", rpt->host->hostname, rpt->client_ip, rpt->client_port);
log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - CANNOT REPLY");
error("STREAM %s [receive from [%s]:%s]: cannot send command.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port);
close(rpt->fd);
return 0;
}
@ -575,14 +590,14 @@ static int rrdpush_receive(struct receiver_state *rpt)
, rpt->hostname
, rpt->client_ip
, rpt->client_port
, rpt->host->hostname
, rrdhost_hostname(rpt->host)
, rpt->host->machine_guid
, rpt->host->rrd_update_every
, rpt->host->rrd_history_entries
, rrd_memory_mode_name(rpt->host->rrd_memory_mode)
, (health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto")
, ssl ? " SSL," : ""
, rpt->host->tags?rpt->host->tags:""
, rrdhost_tags(rpt->host)
);
#endif // NETDATA_INTERNAL_CHECKS
@ -605,7 +620,7 @@ static int rrdpush_receive(struct receiver_state *rpt)
snprintfz(cd.fullfilename, FILENAME_MAX, "%s:%s", rpt->client_ip, rpt->client_port);
snprintfz(cd.cmd, PLUGINSD_CMD_MAX, "%s:%s", rpt->client_ip, rpt->client_port);
info("STREAM %s [receive from [%s]:%s]: initializing communication...", rpt->host->hostname, rpt->client_ip, rpt->client_port);
info("STREAM %s [receive from [%s]:%s]: initializing communication...", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port);
char initial_response[HTTP_HEADER_SIZE];
if (rpt->stream_version > 1) {
if(rpt->stream_version >= STREAM_VERSION_COMPRESSION){
@ -618,13 +633,13 @@ static int rrdpush_receive(struct receiver_state *rpt)
}
#endif
}
info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->stream_version);
info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->stream_version);
sprintf(initial_response, "%s%u", START_STREAMING_PROMPT_VN, rpt->stream_version);
} else if (rpt->stream_version == 1) {
info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->stream_version);
info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->stream_version);
sprintf(initial_response, "%s", START_STREAMING_PROMPT_V2);
} else {
info("STREAM %s [receive from [%s]:%s]: Netdata is using first stream protocol.", rpt->host->hostname, rpt->client_ip, rpt->client_port);
info("STREAM %s [receive from [%s]:%s]: Netdata is using first stream protocol.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port);
sprintf(initial_response, "%s", START_STREAMING_PROMPT);
}
debug(D_STREAM, "Initial response to %s: %s", rpt->client_ip, initial_response);
@ -635,27 +650,27 @@ static int rrdpush_receive(struct receiver_state *rpt)
#else
if(send_timeout(rpt->fd, initial_response, strlen(initial_response), 0, 60) != strlen(initial_response)) {
#endif
log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "FAILED - CANNOT REPLY");
error("STREAM %s [receive from [%s]:%s]: cannot send ready command.", rpt->host->hostname, rpt->client_ip, rpt->client_port);
log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - CANNOT REPLY");
error("STREAM %s [receive from [%s]:%s]: cannot send ready command.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port);
close(rpt->fd);
return 0;
}
// remove the non-blocking flag from the socket
if(sock_delnonblock(rpt->fd) < 0)
error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->fd);
error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd);
struct timeval timeout;
timeout.tv_sec = 120;
timeout.tv_usec = 0;
if (unlikely(setsockopt(rpt->fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout) != 0))
error("STREAM %s [receive from [%s]:%s]: cannot set timeout for socket %d", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->fd);
error("STREAM %s [receive from [%s]:%s]: cannot set timeout for socket %d", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd);
// convert the socket to a FILE *
FILE *fp = fdopen(rpt->fd, "r");
if(!fp) {
log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "FAILED - SOCKET ERROR");
error("STREAM %s [receive from [%s]:%s]: failed to get a FILE for FD %d.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->fd);
log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - SOCKET ERROR");
error("STREAM %s [receive from [%s]:%s]: failed to get a FILE for FD %d.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd);
close(rpt->fd);
return 0;
}
@ -686,7 +701,7 @@ static int rrdpush_receive(struct receiver_state *rpt)
info(
"Postponing health checks for %" PRId64 " seconds, on host '%s', because it was just connected.",
(int64_t)alarms_delay,
rpt->host->hostname);
rrdhost_hostname(rpt->host));
}
}
rpt->host->senders_connect_time = now_realtime_sec();
@ -695,8 +710,8 @@ static int rrdpush_receive(struct receiver_state *rpt)
rrdhost_unlock(rpt->host);
// call the plugins.d processor to receive the metrics
info("STREAM %s [receive from [%s]:%s]: receiving metrics...", rpt->host->hostname, rpt->client_ip, rpt->client_port);
log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "CONNECTED");
info("STREAM %s [receive from [%s]:%s]: receiving metrics...", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port);
log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "CONNECTED");
cd.version = rpt->stream_version;
@ -758,6 +773,7 @@ void *rrdpush_receiver_thread(void *ptr) {
info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
worker_register("STREAMRCV");
worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_READ, "received bytes", "bytes/s", WORKER_METRIC_INCREMENTAL);
rrdpush_receive(rpt);
worker_unregister();

140
streaming/rrdpush.c
View File

@ -140,8 +140,8 @@ static inline int should_send_chart_matching(RRDSET *st) {
if(!rrdset_flag_check(st, RRDSET_FLAG_UPSTREAM_SEND|RRDSET_FLAG_UPSTREAM_IGNORE)) {
RRDHOST *host = st->rrdhost;
if(simple_pattern_matches(host->rrdpush_send_charts_matching, st->id) ||
simple_pattern_matches(host->rrdpush_send_charts_matching, st->name)) {
if(simple_pattern_matches(host->rrdpush_send_charts_matching, rrdset_id(st)) ||
simple_pattern_matches(host->rrdpush_send_charts_matching, rrdset_name(st))) {
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE);
rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_SEND);
}
@ -175,20 +175,17 @@ int configured_as_parent() {
// checks if the current chart definition has been sent
static inline int need_to_send_chart_definition(RRDSET *st) {
rrdset_check_rdlock(st);
if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_UPSTREAM_EXPOSED))))
return 1;
RRDDIM *rd;
rrddim_foreach_read(rd, st) {
dfe_start_read(st->rrddim_root_index, rd) {
if(unlikely(!rd->exposed)) {
#ifdef NETDATA_INTERNAL_CHECKS
info("host '%s', chart '%s', dimension '%s' flag 'exposed' triggered chart refresh to upstream", st->rrdhost->hostname, st->id, rd->id);
#endif
internal_error(true, "host '%s', chart '%s', dimension '%s' flag 'exposed' triggered chart refresh to upstream", rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(rd));
return 1;
}
}
dfe_done(rd);
return 0;
}
@ -216,9 +213,9 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
// properly set the name for the remote end to parse it
char *name = "";
if(likely(st->name)) {
if(unlikely(strcmp(st->id, st->name))) {
if(unlikely(st->id != st->name)) {
// they differ
name = strchr(st->name, '.');
name = strchr(rrdset_name(st), '.');
if(name)
name++;
else
@ -230,12 +227,12 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
buffer_sprintf(
host->sender->build
, "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s %s\" \"%s\" \"%s\"\n"
, st->id
, rrdset_id(st)
, name
, st->title
, st->units
, st->family
, st->context
, rrdset_title(st)
, rrdset_units(st)
, rrdset_family(st)
, rrdset_context(st)
, rrdset_type_name(st->chart_type)
, st->priority
, st->update_every
@ -243,8 +240,8 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
, rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":""
, rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":""
, rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)?"hidden":""
, (st->plugin_name)?st->plugin_name:""
, (st->module_name)?st->module_name:""
, rrdset_plugin_name(st)
, rrdset_module_name(st)
);
// send the chart labels
@ -257,8 +254,8 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
buffer_sprintf(
host->sender->build
, "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"
, rd->id
, rd->name
, rrddim_id(rd)
, rrddim_name(rd)
, rrd_algorithm_name(rd->algorithm)
, rd->multiplier
, rd->divisor
@ -278,7 +275,7 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
buffer_sprintf(
host->sender->build
, "VARIABLE CHART %s = " NETDATA_DOUBLE_FORMAT "\n"
, rs->variable
, string2str(rs->variable)
, *value
);
}
@ -288,40 +285,75 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
}
// sends the current chart dimensions
static inline void rrdpush_send_chart_metrics_nolock(RRDSET *st, struct sender_state *s) {
static inline bool rrdpush_send_chart_metrics_nolock(RRDSET *st, struct sender_state *s) {
RRDHOST *host = st->rrdhost;
buffer_sprintf(host->sender->build, "BEGIN \"%s\" %llu", st->id, (st->last_collected_time.tv_sec > st->upstream_resync_time)?st->usec_since_last_update:0);
buffer_sprintf(host->sender->build, "BEGIN \"%s\" %llu", rrdset_id(st), (st->last_collected_time.tv_sec > st->upstream_resync_time)?st->usec_since_last_update:0);
if (s->version >= VERSION_GAP_FILLING)
buffer_sprintf(host->sender->build, " %"PRId64"\n", (int64_t)st->last_collected_time.tv_sec);
else
buffer_strcat(host->sender->build, "\n");
size_t count_of_dimensions_written = 0;
RRDDIM *rd;
rrddim_foreach_read(rd, st) {
if(rd->updated && rd->exposed)
buffer_sprintf(host->sender->build
, "SET \"%s\" = " COLLECTED_NUMBER_FORMAT "\n"
, rd->id
, rd->collected_value
);
if(rd->updated && rd->exposed) {
buffer_sprintf(host->sender->build, "SET \"%s\" = " COLLECTED_NUMBER_FORMAT "\n", rrddim_id(rd), rd->collected_value);
count_of_dimensions_written++;
}
}
buffer_strcat(host->sender->build, "END\n");
return count_of_dimensions_written != 0;
}
static void rrdpush_sender_thread_spawn(RRDHOST *host);
// Called from the internal collectors to mark a chart obsolete.
void rrdset_push_chart_definition_now(RRDSET *st) {
bool rrdset_push_chart_definition_now(RRDSET *st) {
RRDHOST *host = st->rrdhost;
if(unlikely(!host->rrdpush_send_enabled || !should_send_chart_matching(st)))
return;
return false;
rrdset_rdlock(st);
sender_start(host->sender);
rrdpush_send_chart_definition_nolock(st);
sender_commit(host->sender);
rrdset_unlock(st);
return true;
}
bool rrdpush_incremental_transmission_of_chart_definitions(RRDHOST *host, DICTFE *dictfe, bool restart, bool stop) {
if(stop || restart)
dictionary_foreach_done(dictfe);
if(stop)
return false;
RRDSET *st = NULL;
if(unlikely(!dictfe->dict)) {
st = dictionary_foreach_start_rw(dictfe, host->rrdset_root_index, DICTIONARY_LOCK_REENTRANT);
}
else
st = dictionary_foreach_next(dictfe);
do {
while(st && !need_to_send_chart_definition(st))
st = dictionary_foreach_next(dictfe);
if(st && rrdset_push_chart_definition_now(st))
break;
} while((st = dictionary_foreach_next(dictfe)));
if (!st) {
dictionary_foreach_done(dictfe);
return false;
}
return true;
}
void rrdset_done_push(RRDSET *st) {
@ -334,29 +366,37 @@ void rrdset_done_push(RRDSET *st) {
rrdpush_sender_thread_spawn(host);
// Handle non-connected case
if(unlikely(!__atomic_load_n(&host->rrdpush_sender_connected, __ATOMIC_SEQ_CST))) {
if(unlikely(!__atomic_load_n(&host->rrdpush_sender_connected, __ATOMIC_SEQ_CST)
|| !rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS))) {
if(unlikely(!host->rrdpush_sender_error_shown))
error("STREAM %s [send]: not ready - discarding collected metrics.", host->hostname);
error("STREAM %s [send]: not ready - collected metrics are not sent to parent.", rrdhost_hostname(host));
host->rrdpush_sender_error_shown = 1;
return;
}
else if(unlikely(host->rrdpush_sender_error_shown)) {
info("STREAM %s [send]: sending metrics...", host->hostname);
info("STREAM %s [send]: sending metrics to parent...", rrdhost_hostname(host));
host->rrdpush_sender_error_shown = 0;
}
if(dictionary_stats_entries(st->rrddim_root_index) == 0)
return;
sender_start(host->sender);
if(need_to_send_chart_definition(st))
rrdpush_send_chart_definition_nolock(st);
rrdpush_send_chart_metrics_nolock(st, host->sender);
if(rrdpush_send_chart_metrics_nolock(st, host->sender)) {
// signal the sender there are more data
if (host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1)
error("STREAM %s [send]: cannot write to internal pipe", rrdhost_hostname(host));
// signal the sender there are more data
if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1)
error("STREAM %s [send]: cannot write to internal pipe", host->hostname);
sender_commit(host->sender);
sender_commit(host->sender);
}
else
sender_cancel(host->sender);
}
// labels
@ -376,7 +416,7 @@ void rrdpush_send_labels(RRDHOST *host) {
sender_commit(host->sender);
if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1)
error("STREAM %s [send]: cannot write to internal pipe", host->hostname);
error("STREAM %s [send]: cannot write to internal pipe", rrdhost_hostname(host));
rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
}
@ -399,7 +439,7 @@ void rrdpush_claimed_id(RRDHOST *host)
// signal the sender there are more data
if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1)
error("STREAM %s [send]: cannot write to internal pipe", host->hostname);
error("STREAM %s [send]: cannot write to internal pipe", rrdhost_hostname(host));
}
int connect_to_one_of_destinations(
@ -496,7 +536,7 @@ void rrdpush_sender_thread_stop(RRDHOST *host) {
netdata_thread_t thr = 0;
if(host->rrdpush_sender_spawn) {
info("STREAM %s [send]: signaling sending thread to stop...", host->hostname);
info("STREAM %s [send]: signaling sending thread to stop...", rrdhost_hostname(host));
// signal the thread that we want to join it
host->rrdpush_sender_join = 1;
@ -512,10 +552,10 @@ void rrdpush_sender_thread_stop(RRDHOST *host) {
netdata_mutex_unlock(&host->sender->mutex);
if(thr != 0) {
info("STREAM %s [send]: waiting for the sending thread to stop...", host->hostname);
info("STREAM %s [send]: waiting for the sending thread to stop...", rrdhost_hostname(host));
void *result;
netdata_thread_join(thr, &result);
info("STREAM %s [send]: sending thread has exited.", host->hostname);
info("STREAM %s [send]: sending thread has exited.", rrdhost_hostname(host));
}
}
@ -533,10 +573,10 @@ static void rrdpush_sender_thread_spawn(RRDHOST *host) {
if(!host->rrdpush_sender_spawn) {
char tag[NETDATA_THREAD_TAG_MAX + 1];
snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STREAM_SENDER[%s]", host->hostname);
snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STREAM_SENDER[%s]", rrdhost_hostname(host));
if(netdata_thread_create(&host->rrdpush_sender_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, rrdpush_sender_thread, (void *) host->sender))
error("STREAM %s [send]: failed to create new thread for client.", host->hostname);
error("STREAM %s [send]: failed to create new thread for client.", rrdhost_hostname(host));
else
host->rrdpush_sender_spawn = 1;
}
@ -746,7 +786,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
struct receiver_state *rpt = callocz(1, sizeof(*rpt));
rrd_rdlock();
RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
RRDHOST *host = rrdhost_find_by_guid(machine_guid);
if (unlikely(host && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) /* Ignore archived hosts. */
host = NULL;
if (host) {
@ -763,7 +803,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
info(
"STREAM %s [receive from [%s]:%s]: multiple connections for same host detected - "
"existing connection is dead (%"PRId64" sec), accepting new connection.",
host->hostname,
rrdhost_hostname(host),
w->client_ip,
w->client_port,
(int64_t)age);
@ -772,12 +812,12 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
netdata_mutex_unlock(&host->receiver_lock);
rrdhost_unlock(host);
rrd_unlock();
log_stream_connection(w->client_ip, w->client_port, key, host->machine_guid, host->hostname,
log_stream_connection(w->client_ip, w->client_port, key, host->machine_guid, rrdhost_hostname(host),
"REJECTED - ALREADY CONNECTED");
info(
"STREAM %s [receive from [%s]:%s]: multiple connections for same host detected - "
"existing connection is active (within last %"PRId64" sec), rejecting new connection.",
host->hostname,
rrdhost_hostname(host),
w->client_ip,
w->client_port,
(int64_t)age);

9
streaming/rrdpush.h
View File

@ -46,9 +46,10 @@ typedef struct {
} stream_encoded_t;
#ifdef ENABLE_COMPRESSION
#define LZ4_MAX_MSG_SIZE 0x4000
struct compressor_state {
char *buffer;
size_t buffer_size;
char *compression_result_buffer;
size_t compression_result_buffer_size;
struct compressor_data *data; // Compression API specific data
void (*reset)(struct compressor_state *state);
size_t (*compress)(struct compressor_state *state, const char *data, size_t size, char **buffer);
@ -164,10 +165,12 @@ extern void sender_init(RRDHOST *parent);
extern struct rrdpush_destinations *destinations_init(const char *destinations);
void sender_start(struct sender_state *s);
void sender_commit(struct sender_state *s);
void sender_cancel(struct sender_state *s);
extern int rrdpush_init();
extern int configured_as_parent();
extern void rrdset_done_push(RRDSET *st);
extern void rrdset_push_chart_definition_now(RRDSET *st);
extern bool rrdset_push_chart_definition_now(RRDSET *st);
extern bool rrdpush_incremental_transmission_of_chart_definitions(RRDHOST *host, DICTFE *dictfe, bool restart, bool stop);
extern void *rrdpush_sender_thread(void *ptr);
extern void rrdpush_send_labels(RRDHOST *host);
extern void rrdpush_claimed_id(RRDHOST *host);

307
streaming/sender.c
View File

@ -17,9 +17,12 @@
#define WORKER_SENDER_JOB_DISCONNECT_RECEIVE_ERROR 12
#define WORKER_SENDER_JOB_DISCONNECT_SEND_ERROR 13
#define WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION 14
#define WORKER_SENDER_JOB_BUFFER_RATIO 15
#define WORKER_SENDER_JOB_BYTES_RECEIVED 16
#define WORKER_SENDER_JOB_BYTES_SENT 17
#if WORKER_UTILIZATION_MAX_JOB_TYPES < 15
#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 15
#if WORKER_UTILIZATION_MAX_JOB_TYPES < 18
#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 18
#endif
extern struct config stream_config;
@ -33,6 +36,11 @@ void sender_start(struct sender_state *s) {
buffer_flush(s->build);
}
void sender_cancel(struct sender_state *s) {
buffer_flush(s->build);
netdata_mutex_unlock(&s->mutex);
}
static inline void rrdpush_sender_thread_close_socket(RRDHOST *host);
#ifdef ENABLE_COMPRESSION
@ -43,11 +51,11 @@ static inline void rrdpush_sender_thread_close_socket(RRDHOST *host);
*/
static inline void deactivate_compression(struct sender_state *s) {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION);
error("STREAM_COMPRESSION: Deactivating compression to avoid stream corruption");
error("STREAM_COMPRESSION: Compression returned error, disabling it.");
default_compression_enabled = 0;
s->rrdpush_compression = 0;
s->version = STREAM_VERSION_CLABELS;
error("STREAM_COMPRESSION %s [send to %s]: Restarting connection without compression", s->host->hostname, s->connected_to);
error("STREAM %s [send to %s]: Restarting connection without compression.", rrdhost_hostname(s->host), s->connected_to);
rrdpush_sender_thread_close_socket(s->host);
}
#endif
@ -59,27 +67,55 @@ void sender_commit(struct sender_state *s) {
#ifdef ENABLE_COMPRESSION
if (src && src_len) {
if (s->compressor && s->rrdpush_compression) {
src_len = s->compressor->compress(s->compressor, src, src_len, &src);
if (!src_len) {
deactivate_compression(s);
buffer_flush(s->build);
netdata_mutex_unlock(&s->mutex);
return;
while(src_len) {
size_t size_to_compress = src_len;
if(size_to_compress > LZ4_MAX_MSG_SIZE) {
// we need to find the last newline
// so that the decompressor will have a whole line to work with
const char *t = &src[LZ4_MAX_MSG_SIZE - 1];
while(t-- > src)
if(*t == '\n')
break;
if(t == src)
size_to_compress = LZ4_MAX_MSG_SIZE;
else
size_to_compress = t - src + 1;
}
char *dst;
size_t dst_len = s->compressor->compress(s->compressor, src, size_to_compress, &dst);
if (!dst_len) {
deactivate_compression(s);
buffer_flush(s->build);
netdata_mutex_unlock(&s->mutex);
return;
}
if(cbuffer_add_unsafe(s->host->sender->buffer, dst, dst_len))
s->overflow = 1;
src = src + size_to_compress;
src_len -= size_to_compress;
}
}
if(cbuffer_add_unsafe(s->host->sender->buffer, src, src_len))
else if(cbuffer_add_unsafe(s->host->sender->buffer, src, src_len))
s->overflow = 1;
}
#else
if(cbuffer_add_unsafe(s->host->sender->buffer, src, src_len))
s->overflow = 1;
#endif
buffer_flush(s->build);
netdata_mutex_unlock(&s->mutex);
}
static inline void rrdpush_sender_thread_close_socket(RRDHOST *host) {
rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS);
__atomic_clear(&host->rrdpush_sender_connected, __ATOMIC_SEQ_CST);
if(host->rrdpush_sender_socket != -1) {
@ -94,11 +130,11 @@ static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *ho
buffer_sprintf(
host->sender->build
, "VARIABLE HOST %s = " NETDATA_DOUBLE_FORMAT "\n"
, rv->name
, rrdvar_name(rv)
, *value
);
debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rv->name, *value);
debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rrdvar_name(rv), *value);
}
void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) {
@ -110,7 +146,7 @@ void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) {
}
static int rrdpush_sender_thread_custom_host_variables_callback(void *rrdvar_ptr, void *host_ptr) {
static int rrdpush_sender_thread_custom_host_variables_callback(const char *name __maybe_unused, void *rrdvar_ptr, void *host_ptr) {
RRDVAR *rv = (RRDVAR *)rrdvar_ptr;
RRDHOST *host = (RRDHOST *)host_ptr;
@ -127,7 +163,7 @@ static int rrdpush_sender_thread_custom_host_variables_callback(void *rrdvar_ptr
static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) {
sender_start(host->sender);
int ret = rrdvar_callback_for_all_host_variables(host, rrdpush_sender_thread_custom_host_variables_callback, host);
int ret = rrdvar_walkthrough_read(host->rrdvar_root_index, rrdpush_sender_thread_custom_host_variables_callback, host);
(void)ret;
sender_commit(host->sender);
@ -162,7 +198,7 @@ static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) {
size_t len = cbuffer_next_unsafe(host->sender->buffer, NULL);
if (len)
error("STREAM %s [send]: discarding %zu bytes of metrics already in the buffer.", host->hostname, len);
error("STREAM %s [send]: discarding %zu bytes of metrics already in the buffer.", rrdhost_hostname(host), len);
cbuffer_remove_unsafe(host->sender->buffer, len);
netdata_mutex_unlock(&host->sender->mutex);
@ -259,7 +295,7 @@ static int rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_po
rrdpush_sender_thread_close_socket(host);
debug(D_STREAM, "STREAM: Attempting to connect...");
info("STREAM %s [send to %s]: connecting...", host->hostname, host->rrdpush_send_destination);
info("STREAM %s [send to %s]: connecting...", rrdhost_hostname(host), host->rrdpush_send_destination);
host->rrdpush_sender_socket = connect_to_one_of_destinations(
host->destinations
@ -272,11 +308,11 @@ static int rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_po
);
if(unlikely(host->rrdpush_sender_socket == -1)) {
error("STREAM %s [send to %s]: failed to connect", host->hostname, host->rrdpush_send_destination);
error("STREAM %s [send to %s]: failed to connect", rrdhost_hostname(host), host->rrdpush_send_destination);
return 0;
}
info("STREAM %s [send to %s]: initializing communication...", host->hostname, s->connected_to);
info("STREAM %s [send to %s]: initializing communication...", rrdhost_hostname(host), s->connected_to);
#ifdef ENABLE_HTTPS
if( netdata_client_ctx ){
@ -370,19 +406,19 @@ if(!s->rrdpush_compression)
"User-Agent: %s/%s\r\n"
"Accept: */*\r\n\r\n"
, host->rrdpush_send_api_key
, host->hostname
, host->registry_hostname
, rrdhost_hostname(host)
, rrdhost_registry_hostname(host)
, host->machine_guid
, default_rrd_update_every
, host->os
, host->timezone
, host->abbrev_timezone
, rrdhost_os(host)
, rrdhost_timezone(host)
, rrdhost_abbrev_timezone(host)
, host->utc_offset
, host->system_info->hops + 1
, host->system_info->ml_capable
, host->system_info->ml_enabled
, host->system_info->mc_version
, (host->tags) ? host->tags : ""
, rrdhost_tags(host)
, s->version
, (host->system_info->cloud_provider_type) ? host->system_info->cloud_provider_type : ""
, (host->system_info->cloud_instance_type) ? host->system_info->cloud_instance_type : ""
@ -412,8 +448,8 @@ if(!s->rrdpush_compression)
, (host->system_info->host_ram_total) ? host->system_info->host_ram_total : ""
, (host->system_info->host_disk_space) ? host->system_info->host_disk_space : ""
, STREAMING_PROTOCOL_VERSION
, host->program_name
, host->program_version
, rrdhost_program_name(host)
, rrdhost_program_version(host)
);
http[eol] = 0x00;
rrdpush_clean_encoded(&se);
@ -456,12 +492,12 @@ if(!s->rrdpush_compression)
if(send_timeout(host->rrdpush_sender_socket, http, strlen(http), 0, timeout) == -1) {
#endif
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", host->hostname, s->connected_to);
error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", rrdhost_hostname(host), s->connected_to);
rrdpush_sender_thread_close_socket(host);
return 0;
}
info("STREAM %s [send to %s]: waiting response from remote netdata...", host->hostname, s->connected_to);
info("STREAM %s [send to %s]: waiting response from remote netdata...", rrdhost_hostname(host), s->connected_to);
ssize_t received;
#ifdef ENABLE_HTTPS
@ -472,7 +508,7 @@ if(!s->rrdpush_compression)
if(received == -1) {
#endif
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
error("STREAM %s [send to %s]: remote netdata does not respond.", host->hostname, s->connected_to);
error("STREAM %s [send to %s]: remote netdata does not respond.", rrdhost_hostname(host), s->connected_to);
rrdpush_sender_thread_close_socket(host);
return 0;
}
@ -482,7 +518,7 @@ if(!s->rrdpush_compression)
int32_t version = (int32_t)parse_stream_version(host, http);
if(version == -1) {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE);
error("STREAM %s [send to %s]: server is not replying properly (is it a netdata?).", host->hostname, s->connected_to);
error("STREAM %s [send to %s]: server is not replying properly (is it a netdata?).", rrdhost_hostname(host), s->connected_to);
rrdpush_sender_thread_close_socket(host);
//catch other reject reasons and force to check other destinations
if (host->destination->next)
@ -490,19 +526,19 @@ if(!s->rrdpush_compression)
return 0;
}
else if(version == -2) {
error("STREAM %s [send to %s]: remote server is the localhost for [%s].", host->hostname, s->connected_to, host->hostname);
error("STREAM %s [send to %s]: remote server is the localhost for [%s].", rrdhost_hostname(host), s->connected_to, rrdhost_hostname(host));
rrdpush_sender_thread_close_socket(host);
host->destination->disabled_because_of_localhost = 1;
return 0;
}
else if(version == -3) {
error("STREAM %s [send to %s]: remote server already receives metrics for [%s].", host->hostname, s->connected_to, host->hostname);
error("STREAM %s [send to %s]: remote server already receives metrics for [%s].", rrdhost_hostname(host), s->connected_to, rrdhost_hostname(host));
rrdpush_sender_thread_close_socket(host);
host->destination->disabled_already_streaming = now_realtime_sec();
return 0;
}
else if(version == -4) {
error("STREAM %s [send to %s]: remote server denied access for [%s].", host->hostname, s->connected_to, host->hostname);
error("STREAM %s [send to %s]: remote server denied access for [%s].", rrdhost_hostname(host), s->connected_to, rrdhost_hostname(host));
rrdpush_sender_thread_close_socket(host);
if (host->destination->next)
host->destination->disabled_because_of_denied_access = 1;
@ -520,23 +556,23 @@ if(!s->rrdpush_compression)
}
else {
//parent does not support compression or has compression disabled
debug(D_STREAM, "Stream is uncompressed! One of the agents (%s <-> %s) does not support compression OR compression is disabled.", s->connected_to, s->host->hostname);
infoerr("Stream is uncompressed! One of the agents (%s <-> %s) does not support compression OR compression is disabled.", s->connected_to, s->host->hostname);
debug(D_STREAM, "Stream is uncompressed! One of the agents (%s <-> %s) does not support compression OR compression is disabled.", s->connected_to, rrdhost_hostname(s->host));
infoerr("Stream is uncompressed! One of the agents (%s <-> %s) does not support compression OR compression is disabled.", s->connected_to, rrdhost_hostname(s->host));
s->version = STREAM_VERSION_CLABELS;
}
#endif //ENABLE_COMPRESSION
info("STREAM %s [send to %s]: established communication with a parent using protocol version %d - ready to send metrics..."
, host->hostname
info("STREAM %s [send to %s]: established communication with a parent using protocol version %d"
, rrdhost_hostname(host)
, s->connected_to
, s->version);
if(sock_setnonblock(host->rrdpush_sender_socket) < 0)
error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", host->hostname, s->connected_to);
error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", rrdhost_hostname(host), s->connected_to);
if(sock_enlarge_out(host->rrdpush_sender_socket) < 0)
error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", host->hostname, s->connected_to);
error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", rrdhost_hostname(host), s->connected_to);
debug(D_STREAM, "STREAM: Connected on fd %d...", host->rrdpush_sender_socket);
@ -578,7 +614,8 @@ static void attempt_to_connect(struct sender_state *state)
}
// TCP window is open and we have data to transmit.
void attempt_to_send(struct sender_state *s) {
static ssize_t attempt_to_send(struct sender_state *s) {
ssize_t ret = 0;
rrdpush_send_labels(s->host);
@ -591,42 +628,44 @@ void attempt_to_send(struct sender_state *s) {
char *chunk;
size_t outstanding = cbuffer_next_unsafe(s->buffer, &chunk);
debug(D_STREAM, "STREAM: Sending data. Buffer r=%zu w=%zu s=%zu, next chunk=%zu", cb->read, cb->write, cb->size, outstanding);
ssize_t ret;
#ifdef ENABLE_HTTPS
SSL *conn = s->host->ssl.conn ;
if(conn && !s->host->ssl.flags) {
if(conn && !s->host->ssl.flags)
ret = SSL_write(conn, chunk, outstanding);
} else {
else
ret = send(s->host->rrdpush_sender_socket, chunk, outstanding, MSG_DONTWAIT);
}
#else
ret = send(s->host->rrdpush_sender_socket, chunk, outstanding, MSG_DONTWAIT);
#endif
if (likely(ret > 0)) {
cbuffer_remove_unsafe(s->buffer, ret);
s->sent_bytes_on_this_connection += ret;
s->sent_bytes += ret;
debug(D_STREAM, "STREAM %s [send to %s]: Sent %zd bytes", s->host->hostname, s->connected_to, ret);
debug(D_STREAM, "STREAM %s [send to %s]: Sent %zd bytes", rrdhost_hostname(s->host), s->connected_to, ret);
s->last_sent_t = now_monotonic_sec();
}
else if (ret == -1 && (errno == EAGAIN || errno == EINTR || errno == EWOULDBLOCK))
debug(D_STREAM, "STREAM %s [send to %s]: unavailable after polling POLLOUT", s->host->hostname, s->connected_to);
debug(D_STREAM, "STREAM %s [send to %s]: unavailable after polling POLLOUT", rrdhost_hostname(s->host), s->connected_to);
else if (ret == -1) {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SEND_ERROR);
debug(D_STREAM, "STREAM: Send failed - closing socket...");
error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", s->host->hostname, s->connected_to, s->sent_bytes_on_this_connection);
error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", rrdhost_hostname(s->host), s->connected_to, s->sent_bytes_on_this_connection);
rrdpush_sender_thread_close_socket(s->host);
}
else {
else
debug(D_STREAM, "STREAM: send() returned 0 -> no error but no transmission");
}
netdata_mutex_unlock(&s->mutex);
netdata_thread_enable_cancelability();
return ret;
}
void attempt_read(struct sender_state *s) {
int ret;
static ssize_t attempt_read(struct sender_state *s) {
ssize_t ret = 0;
#ifdef ENABLE_HTTPS
if (s->host->ssl.conn && !s->host->stream_ssl.flags) {
ERR_clear_error();
@ -634,44 +673,46 @@ int ret;
ret = SSL_read(s->host->ssl.conn, s->read_buffer, desired);
if (ret > 0 ) {
s->read_len += ret;
return;
return ret;
}
int sslerrno = SSL_get_error(s->host->ssl.conn, desired);
if (sslerrno == SSL_ERROR_WANT_READ || sslerrno == SSL_ERROR_WANT_WRITE)
return;
return ret;
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);
u_long err;
char buf[256];
while ((err = ERR_get_error()) != 0) {
ERR_error_string_n(err, buf, sizeof(buf));
error("STREAM %s [send to %s] ssl error: %s", s->host->hostname, s->connected_to, buf);
error("STREAM %s [send to %s] ssl error: %s", rrdhost_hostname(s->host), s->connected_to, buf);
}
error("Restarting connection");
rrdpush_sender_thread_close_socket(s->host);
return;
return ret;
}
#endif
ret = recv(s->host->rrdpush_sender_socket, s->read_buffer + s->read_len, sizeof(s->read_buffer) - s->read_len - 1,MSG_DONTWAIT);
if (ret>0) {
if (ret > 0) {
s->read_len += ret;
return;
return ret;
}
debug(D_STREAM, "Socket was POLLIN, but req %zu bytes gave %d", sizeof(s->read_buffer) - s->read_len - 1, ret);
debug(D_STREAM, "Socket was POLLIN, but req %zu bytes gave %zd", sizeof(s->read_buffer) - s->read_len - 1, ret);
if (ret<0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR))
return;
if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR))
return ret;
if (ret==0) {
if (ret == 0) {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_PARENT_CLOSED);
error("STREAM %s [send to %s]: connection closed by far end. Restarting connection", s->host->hostname, s->connected_to);
error("STREAM %s [send to %s]: connection closed by far end. Restarting connection", rrdhost_hostname(s->host), s->connected_to);
}
else {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_RECEIVE_ERROR);
error("STREAM %s [send to %s]: error during receive (%d). Restarting connection", s->host->hostname, s->connected_to, ret);
error("STREAM %s [send to %s]: error during receive (%zd). Restarting connection", rrdhost_hostname(s->host), s->connected_to, ret);
}
rrdpush_sender_thread_close_socket(s->host);
return ret;
}
// This is just a placeholder until the gap filling state machine is inserted
@ -680,7 +721,7 @@ void execute_commands(struct sender_state *s) {
*end = 0;
while( start<end && (newline=strchr(start, '\n')) ) {
*newline = 0;
info("STREAM %s [send to %s] received command over connection: %s", s->host->hostname, s->connected_to, start);
info("STREAM %s [send to %s] received command over connection: %s", rrdhost_hostname(s->host), s->connected_to, start);
start = newline+1;
}
if (start<end) {
@ -689,15 +730,57 @@ void execute_commands(struct sender_state *s) {
}
}
struct rrdpush_sender_thread_data {
struct sender_state *sender_state;
RRDHOST *host;
DICTFE dictfe;
enum {
SENDING_DEFINITIONS_RESTART,
SENDING_DEFINITIONS_CONTINUE,
SENDING_DEFINITIONS_DONE,
} sending_definitions_status;
};
static size_t cbuffer_available_bytes_with_lock(struct rrdpush_sender_thread_data *thread_data) {
netdata_mutex_lock(&thread_data->sender_state->mutex);
size_t outstanding = cbuffer_available_size_unsafe(thread_data->sender_state->host->sender->buffer);
netdata_mutex_unlock(&thread_data->sender_state->mutex);
return outstanding;
}
static void rrdpush_queue_incremental_definitions(struct rrdpush_sender_thread_data *thread_data) {
while(__atomic_load_n(&thread_data->host->rrdpush_sender_connected, __ATOMIC_SEQ_CST)
&& thread_data->sending_definitions_status != SENDING_DEFINITIONS_DONE
&& cbuffer_available_bytes_with_lock(thread_data) > (thread_data->sender_state->buffer->max_size / 2)) {
if(thread_data->sending_definitions_status == SENDING_DEFINITIONS_RESTART)
info("STREAM %s [send to %s]: sending metric definitions...", rrdhost_hostname(thread_data->host), thread_data->sender_state->connected_to);
bool more_defs_available = rrdpush_incremental_transmission_of_chart_definitions(
thread_data->sender_state->host, &thread_data->dictfe,
thread_data->sending_definitions_status == SENDING_DEFINITIONS_RESTART, false);
if (unlikely(!more_defs_available)) {
thread_data->sending_definitions_status = SENDING_DEFINITIONS_DONE;
info("STREAM %s [send to %s]: sending metric definitions finished.", rrdhost_hostname(thread_data->host), thread_data->sender_state->connected_to);
}
else
thread_data->sending_definitions_status = SENDING_DEFINITIONS_CONTINUE;
}
}
static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
struct rrdpush_sender_thread_data *data = ptr;
worker_unregister();
RRDHOST *host = (RRDHOST *)ptr;
RRDHOST *host = data->host;
rrdpush_incremental_transmission_of_chart_definitions(host, &data->dictfe, false, true);
netdata_mutex_lock(&host->sender->mutex);
info("STREAM %s [send]: sending thread cleans up...", host->hostname);
info("STREAM %s [send]: sending thread cleans up...", rrdhost_hostname(host));
rrdpush_sender_thread_close_socket(host);
@ -713,15 +796,17 @@ static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
}
if(!host->rrdpush_sender_join) {
info("STREAM %s [send]: sending thread detaches itself.", host->hostname);
info("STREAM %s [send]: sending thread detaches itself.", rrdhost_hostname(host));
netdata_thread_detach(netdata_thread_self());
}
host->rrdpush_sender_spawn = 0;
info("STREAM %s [send]: sending thread now exits.", host->hostname);
info("STREAM %s [send]: sending thread now exits.", rrdhost_hostname(host));
netdata_mutex_unlock(&host->sender->mutex);
freez(data);
}
void sender_init(RRDHOST *parent)
@ -749,7 +834,7 @@ void *rrdpush_sender_thread(void *ptr) {
!*s->host->rrdpush_send_destination || !s->host->rrdpush_send_api_key ||
!*s->host->rrdpush_send_api_key) {
error("STREAM %s [send]: thread created (task id %d), but host has streaming disabled.",
s->host->hostname, s->task_id);
rrdhost_hostname(s->host), s->task_id);
return NULL;
}
@ -760,7 +845,7 @@ void *rrdpush_sender_thread(void *ptr) {
}
#endif
info("STREAM %s [send]: thread created (task id %d)", s->host->hostname, s->task_id);
info("STREAM %s [send]: thread created (task id %d)", rrdhost_hostname(s->host), s->task_id);
s->timeout = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "timeout seconds", 60);
s->default_port = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "default port", 19999);
@ -774,9 +859,10 @@ void *rrdpush_sender_thread(void *ptr) {
remote_clock_resync_iterations); // TODO: REMOVE FOR SLEW / GAPFILLING
// initialize rrdpush globals
rrdhost_flag_clear(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS);
__atomic_clear(&s->host->rrdpush_sender_connected, __ATOMIC_SEQ_CST);
if(pipe(s->host->rrdpush_sender_pipe) == -1) {
error("STREAM %s [send]: cannot create required pipe. DISABLING STREAMING THREAD", s->host->hostname);
error("STREAM %s [send]: cannot create required pipe. DISABLING STREAMING THREAD", rrdhost_hostname(s->host));
return NULL;
}
s->version = STREAMING_PROTOCOL_CURRENT_VERSION;
@ -808,7 +894,17 @@ void *rrdpush_sender_thread(void *ptr) {
worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION, "disconnect no compression");
worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE, "disconnect bad handshake");
netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, s->host);
worker_register_job_custom_metric(WORKER_SENDER_JOB_BUFFER_RATIO, "used buffer ratio", "%", WORKER_METRIC_ABSOLUTE);
worker_register_job_custom_metric(WORKER_SENDER_JOB_BYTES_RECEIVED, "bytes received", "bytes/s", WORKER_METRIC_INCREMENTAL);
worker_register_job_custom_metric(WORKER_SENDER_JOB_BYTES_SENT, "bytes sent", "bytes/s", WORKER_METRIC_INCREMENTAL);
struct rrdpush_sender_thread_data *thread_data = callocz(1, sizeof(struct rrdpush_sender_thread_data));
thread_data->sender_state = s;
thread_data->host = s->host;
thread_data->sending_definitions_status = SENDING_DEFINITIONS_RESTART;
netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, thread_data);
for(; s->host->rrdpush_send_enabled && !netdata_exit ;) {
// check for outstanding cancellation requests
netdata_thread_testcancel();
@ -816,6 +912,8 @@ void *rrdpush_sender_thread(void *ptr) {
// The connection attempt blocks (after which we use the socket in nonblocking)
if(unlikely(s->host->rrdpush_sender_socket == -1)) {
worker_is_busy(WORKER_SENDER_JOB_CONNECT);
thread_data->sending_definitions_status = SENDING_DEFINITIONS_RESTART;
rrdhost_flag_clear(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS);
s->overflow = 0;
s->read_len = 0;
s->buffer->read = 0;
@ -828,17 +926,27 @@ void *rrdpush_sender_thread(void *ptr) {
sender_commit(s);
}
rrdpush_claimed_id(s->host);
// TO PUSH METRICS WITH DEFINITIONS:
//if(unlikely(s->host->rrdpush_sender_socket != -1 && __atomic_load_n(&s->host->rrdpush_sender_connected, __ATOMIC_SEQ_CST))) {
// thread_data->sending_definitions_status = SENDING_DEFINITIONS_DONE;
// rrdhost_flag_set(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS);
//}
continue;
}
// If the TCP window never opened then something is wrong, restart connection
if(unlikely(now_monotonic_sec() - s->last_sent_t > s->timeout)) {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", s->host->hostname, s->connected_to, s->timeout, s->sent_bytes_on_this_connection, s->send_attempts);
error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", rrdhost_hostname(s->host), s->connected_to, s->timeout, s->sent_bytes_on_this_connection, s->send_attempts);
rrdpush_sender_thread_close_socket(s->host);
continue;
}
if(unlikely(thread_data->sending_definitions_status != SENDING_DEFINITIONS_DONE))
rrdpush_queue_incremental_definitions(thread_data);
worker_is_idle();
// Wait until buffer opens in the socket or a rrdset_done_push wakes us
@ -847,16 +955,27 @@ void *rrdpush_sender_thread(void *ptr) {
fds[Socket].fd = s->host->rrdpush_sender_socket;
netdata_mutex_lock(&s->mutex);
char *chunk;
size_t outstanding = cbuffer_next_unsafe(s->host->sender->buffer, &chunk);
chunk = NULL; // Do not cache pointer outside of region - could be invalidated
size_t outstanding = cbuffer_next_unsafe(s->host->sender->buffer, NULL);
size_t available = cbuffer_available_size_unsafe(s->host->sender->buffer);
netdata_mutex_unlock(&s->mutex);
worker_set_metric(WORKER_SENDER_JOB_BUFFER_RATIO, (NETDATA_DOUBLE)(s->host->sender->buffer->max_size - available) * 100.0 / (NETDATA_DOUBLE)s->host->sender->buffer->max_size);
if(outstanding) {
s->send_attempts++;
fds[Socket].events = POLLIN | POLLOUT;
}
else {
fds[Socket].events = POLLIN;
if(unlikely(thread_data->sending_definitions_status == SENDING_DEFINITIONS_DONE
&& __atomic_load_n(&s->host->rrdpush_sender_connected, __ATOMIC_SEQ_CST)
&& !rrdhost_flag_check(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS)
)) {
// let the data collection threads know we are ready to push metrics
rrdhost_flag_set(s->host, RRDHOST_FLAG_STREAM_COLLECTED_METRICS);
info("STREAM %s [send to %s]: enabling metrics streaming...", rrdhost_hostname(s->host), s->connected_to);
}
}
int retval = poll(fds, 2, 1000);
@ -874,7 +993,7 @@ void *rrdpush_sender_thread(void *ptr) {
// Only errors from poll() are internal, but try restarting the connection
if(unlikely(retval == -1)) {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_POLL_ERROR);
error("STREAM %s [send to %s]: failed to poll(). Closing socket.", s->host->hostname, s->connected_to);
error("STREAM %s [send to %s]: failed to poll(). Closing socket.", rrdhost_hostname(s->host), s->connected_to);
rrdpush_sender_thread_close_socket(s->host);
continue;
}
@ -884,28 +1003,34 @@ void *rrdpush_sender_thread(void *ptr) {
worker_is_busy(WORKER_SENDER_JOB_PIPE_READ);
debug(D_STREAM, "STREAM: Data added to send buffer (current buffer chunk %zu bytes)...", outstanding);
char buffer[1000 + 1];
if (read(s->host->rrdpush_sender_pipe[PIPE_READ], buffer, 1000) == -1)
error("STREAM %s [send to %s]: cannot read from internal pipe.", s->host->hostname, s->connected_to);
char buffer[10000 + 1];
if (read(s->host->rrdpush_sender_pipe[PIPE_READ], buffer, 10000) == -1)
error("STREAM %s [send to %s]: cannot read from internal pipe.", rrdhost_hostname(s->host), s->connected_to);
}
// Read as much as possible to fill the buffer, split into full lines for execution.
if (fds[Socket].revents & POLLIN) {
worker_is_busy(WORKER_SENDER_JOB_SOCKET_RECEIVE);
attempt_read(s);
ssize_t bytes = attempt_read(s);
if(bytes > 0)
worker_set_metric(WORKER_SENDER_JOB_BYTES_RECEIVED, bytes);
}
worker_is_busy(WORKER_SENDER_JOB_EXECUTE);
execute_commands(s);
if(unlikely(s->read_len)) {
worker_is_busy(WORKER_SENDER_JOB_EXECUTE);
execute_commands(s);
}
// If we have data and have seen the TCP window open then try to close it by a transmission.
if (outstanding && fds[Socket].revents & POLLOUT) {
if(likely(outstanding && fds[Socket].revents & POLLOUT)) {
worker_is_busy(WORKER_SENDER_JOB_SOCKET_SEND);
attempt_to_send(s);
ssize_t bytes = attempt_to_send(s);
if(bytes > 0)
worker_set_metric(WORKER_SENDER_JOB_BYTES_SENT, bytes);
}
// TODO-GAPS - why do we only check this on the socket, not the pipe?
if (outstanding) {
if(outstanding) {
char *error = NULL;
if (unlikely(fds[Socket].revents & POLLERR))
error = "socket reports errors (POLLERR)";
@ -915,18 +1040,18 @@ void *rrdpush_sender_thread(void *ptr) {
error = "connection is invalid (POLLNVAL)";
if(unlikely(error)) {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SOCKER_ERROR);
error("STREAM %s [send to %s]: restart stream because %s - %zu bytes transmitted.", s->host->hostname,
error("STREAM %s [send to %s]: restart stream because %s - %zu bytes transmitted.", rrdhost_hostname(s->host),
s->connected_to, error, s->sent_bytes_on_this_connection);
rrdpush_sender_thread_close_socket(s->host);
}
}
// protection from overflow
if (s->overflow) {
if(unlikely(s->overflow)) {
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW);
errno = 0;
error("STREAM %s [send to %s]: buffer full (%zu-bytes) after %zu bytes. Restarting connection",
s->host->hostname, s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection);
error("STREAM %s [send to %s]: buffer full (allocated %zu bytes) after sending %zu bytes. Restarting connection",
rrdhost_hostname(s->host), s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection);
rrdpush_sender_thread_close_socket(s->host);
}
}

6
web/api/badges/web_buffer_svg.c
View File

@ -1020,19 +1020,19 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
label = dim;
}
else
label = st->name;
label = rrdset_name(st);
}
if(!units) {
if(alarm) {
if(rc->units)
units = rc->units;
units = rrdcalc_units(rc);
else
units = "";
}
else if(options & RRDR_OPTION_PERCENTAGE)
units = "%";
else
units = st->units;
units = rrdset_units(st);
}
debug(D_WEB_CLIENT, "%llu: API command 'badge.svg' for chart '%s', alarm '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', options '0x%08x'"

40
web/api/exporters/shell/allmetrics_shell.c
View File

@ -30,14 +30,14 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
// for each chart
RRDSET *st;
rrdset_foreach_read(st, host) {
if (filter && !simple_pattern_matches(filter, st->name))
if (filter && !simple_pattern_matches(filter, rrdset_name(st)))
continue;
NETDATA_DOUBLE total = 0.0;
char chart[SHELL_ELEMENT_MAX + 1];
shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX);
shell_name_copy(chart, st->name?rrdset_name(st):rrdset_id(st), SHELL_ELEMENT_MAX);
buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name);
buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", rrdset_id(st), rrdset_name(st));
if(rrdset_is_available_for_viewers(st)) {
rrdset_rdlock(st);
@ -46,23 +46,23 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
rrddim_foreach_read(rd, st) {
if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
char dimension[SHELL_ELEMENT_MAX + 1];
shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX);
shell_name_copy(dimension, rd->name?rrddim_name(rd):rrddim_id(rd), SHELL_ELEMENT_MAX);
NETDATA_DOUBLE n = rd->last_stored_value;
if(isnan(n) || isinf(n))
buffer_sprintf(wb, "NETDATA_%s_%s=\"\" # %s\n", chart, dimension, st->units);
buffer_sprintf(wb, "NETDATA_%s_%s=\"\" # %s\n", chart, dimension, rrdset_units(st));
else {
if(rd->multiplier < 0 || rd->divisor < 0) n = -n;
n = roundndd(n);
if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n;
buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, dimension, n, st->units);
buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, dimension, n, rrdset_units(st));
}
}
}
total = roundndd(total);
buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, total, st->units);
buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, total, rrdset_units(st));
rrdset_unlock(st);
}
}
@ -70,22 +70,22 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n");
RRDCALC *rc;
for(rc = host->alarms; rc ;rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
if(!rc->rrdset) continue;
char chart[SHELL_ELEMENT_MAX + 1];
shell_name_copy(chart, rc->rrdset->name?rc->rrdset->name:rc->rrdset->id, SHELL_ELEMENT_MAX);
shell_name_copy(chart, rc->rrdset->name?rrdset_name(rc->rrdset):rrdset_id(rc->rrdset), SHELL_ELEMENT_MAX);
char alarm[SHELL_ELEMENT_MAX + 1];
shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX);
shell_name_copy(alarm, rrdcalc_name(rc), SHELL_ELEMENT_MAX);
NETDATA_DOUBLE n = rc->value;
if(isnan(n) || isinf(n))
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\" # %s\n", chart, alarm, rc->units);
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\" # %s\n", chart, alarm, rrdcalc_units(rc));
else {
n = roundndd(n);
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, alarm, n, rc->units);
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, alarm, n, rrdcalc_units(rc));
}
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status));
@ -110,7 +110,7 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
// for each chart
RRDSET *st;
rrdset_foreach_read(st, host) {
if (filter && !(simple_pattern_matches(filter, st->id) || simple_pattern_matches(filter, st->name)))
if (filter && !(simple_pattern_matches(filter, rrdset_id(st)) || simple_pattern_matches(filter, rrdset_name(st))))
continue;
if(rrdset_is_available_for_viewers(st)) {
@ -127,11 +127,11 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
"\t\t\"last_updated\": %"PRId64",\n"
"\t\t\"dimensions\": {",
chart_counter ? "," : "",
st->id,
st->name,
st->family,
st->context,
st->units,
rrdset_id(st),
rrdset_name(st),
rrdset_family(st),
rrdset_context(st),
rrdset_units(st),
(int64_t)rrdset_last_entry_t_nolock(st));
chart_counter++;
@ -148,8 +148,8 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
"\t\t\t\t\"name\": \"%s\",\n"
"\t\t\t\t\"value\": ",
dimension_counter ? "," : "",
rd->id,
rd->name);
rrddim_id(rd),
rrddim_name(rd));
if(isnan(rd->last_stored_value))
buffer_strcat(wb, "null");

24
web/api/formatters/charts2json.c
View File

@ -57,11 +57,11 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
",\n\t\"memory_mode\": \"%s\""
",\n\t\"custom_info\": \"%s\""
",\n\t\"charts\": {"
, host->hostname
, host->program_version
, rrdhost_hostname(host)
, rrdhost_program_version(host)
, get_release_channel()
, host->os
, host->timezone
, rrdhost_os(host)
, rrdhost_timezone(host)
, host->rrd_update_every
, host->rrd_history_entries
, rrd_memory_mode_name(host->rrd_memory_mode)
@ -74,7 +74,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) {
if(c) buffer_strcat(wb, ",");
buffer_strcat(wb, "\n\t\t\"");
buffer_strcat(wb, st->id);
buffer_strcat(wb, rrdset_id(st));
buffer_strcat(wb, "\": ");
rrdset2json(st, wb, &dimensions, &memory, skip_volatile);
@ -84,7 +84,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
}
RRDCALC *rc;
for(rc = host->alarms; rc ; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
if(rc->rrdset)
alarms++;
}
@ -117,7 +117,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
"\n\t\t\t\"hostname\": \"%s\""
"\n\t\t}"
, (found > 0) ? "," : ""
, h->hostname
, rrdhost_hostname(h)
);
found++;
@ -131,7 +131,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
, "\n\t\t{"
"\n\t\t\t\"hostname\": \"%s\""
"\n\t\t}"
, host->hostname
, rrdhost_hostname(host)
);
}
@ -141,8 +141,8 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
// generate collectors list for the api/v1/info call
struct collector {
char *plugin;
char *module;
const char *plugin;
const char *module;
};
struct array_printer {
@ -176,8 +176,8 @@ void chartcollectors2json(RRDHOST *host, BUFFER *wb) {
rrdset_foreach_read(st, host) {
if (rrdset_is_available_for_viewers(st)) {
struct collector col = {
.plugin = st->plugin_name ? st->plugin_name : "",
.module = st->module_name ? st->module_name : ""
.plugin = rrdset_plugin_name(st),
.module = rrdset_module_name(st)
};
sprintf(name, "%s:%s", col.plugin, col.module);
dictionary_set(dict, name, &col, sizeof(struct collector));

2
web/api/formatters/csv/csv.c
View File

@ -23,7 +23,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
}
buffer_strcat(wb, separator);
if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
buffer_strcat(wb, d->name);
buffer_strcat(wb, rrddim_name(d));
if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
i++;
}

4
web/api/formatters/json/json.c
View File

@ -121,7 +121,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
buffer_fast_strcat(wb, pre_label, pre_label_len);
buffer_strcat(wb, rd->name);
buffer_strcat(wb, rrddim_name(rd));
// buffer_strcat(wb, ".");
// buffer_strcat(wb, rd->rrdset->name);
buffer_fast_strcat(wb, post_label, post_label_len);
@ -253,7 +253,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
buffer_fast_strcat(wb, pre_value, pre_value_len);
if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
buffer_sprintf(wb, "%s%s%s: ", kq, rrddim_name(rd), kq);
if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) {
if(unlikely(options & RRDR_OPTION_NULL2ZERO))

20
web/api/formatters/json_wrapper.c
View File

@ -79,8 +79,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
" %sgroup%s: %s%s%s,\n"
" %soptions%s: %s"
, kq, kq
, kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq
, kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq
, kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_id(r->st), sq
, kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_name(r->st), sq
, kq, kq, r->update_every
, kq, kq, r->st->update_every
, kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t_nolock(r->st))
@ -103,13 +103,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if(i) buffer_strcat(wb, ", ");
buffer_strcat(wb, sq);
buffer_strcat(wb, rd->name);
buffer_strcat(wb, rrddim_name(rd));
buffer_strcat(wb, sq);
i++;
}
if(!i) {
#ifdef NETDATA_INTERNAL_CHECKS
error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", rrdset_id(r->st), r->d, options);
#endif
rows = 0;
buffer_strcat(wb, sq);
@ -127,7 +127,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if(i) buffer_strcat(wb, ", ");
buffer_strcat(wb, sq);
buffer_strcat(wb, rd->id);
buffer_strcat(wb, rrddim_id(rd));
buffer_strcat(wb, sq);
i++;
}
@ -149,8 +149,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name);
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name);
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrddim_id(rd), rrddim_name(rd));
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrddim_id(rd), rrddim_name(rd));
dictionary_set(dict, name, output, len+1);
}
dictionary_walkthrough_read(dict, value_list_output, &co);
@ -160,8 +160,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name);
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name);
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset));
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset));
dictionary_set(dict, name, output, len + 1);
}
@ -198,7 +198,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if (i)
buffer_strcat(wb, ", ");
buffer_strcat(wb, sq);
buffer_strcat(wb, rd->rrdset->id);
buffer_strcat(wb, rrdset_id(rd->rrdset));
buffer_strcat(wb, sq);
i++;
}

12
web/api/formatters/rrd2json.c
View File

@ -8,13 +8,15 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc
if (unlikely(!temp_rd))
return;
onewayalloc_freez(owa, (char *)temp_rd->id);
string_freez(temp_rd->id);
string_freez(temp_rd->name);
if (unlikely(archive_mode)) {
temp_rd->rrdset->counter--;
if (!temp_rd->rrdset->counter) {
onewayalloc_freez(owa, (char *)temp_rd->rrdset->name);
onewayalloc_freez(owa, temp_rd->rrdset->context);
string_freez(temp_rd->rrdset->id);
string_freez(temp_rd->rrdset->name);
string_freez(temp_rd->rrdset->context);
onewayalloc_freez(owa, temp_rd->rrdset);
}
}
@ -111,8 +113,8 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis
rrddim_foreach_read(rd1, st) {
RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM));
rd->id = onewayalloc_strdupz(owa, rd1->id);
rd->name = onewayalloc_strdupz(owa, rd1->name);
rd->id = string_dup(rd1->id);
rd->name = string_dup(rd1->name);
for(int tier = 0; tier < storage_tiers ;tier++) {
if(rd1->tiers[tier])
rd->tiers[tier] = onewayalloc_memdupz(owa, rd1->tiers[tier], sizeof(*rd->tiers[tier]));

30
web/api/formatters/rrdset2json.c
View File

@ -45,18 +45,18 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
"\t\t\t\"units\": \"%s\",\n"
"\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
"\t\t\t\"chart_type\": \"%s\",\n",
st->id,
st->name,
st->type,
st->family,
st->context,
st->title,
st->name,
rrdset_id(st),
rrdset_name(st),
rrdset_type(st),
rrdset_family(st),
rrdset_context(st),
rrdset_title(st),
rrdset_name(st),
st->priority,
st->plugin_name ? st->plugin_name : "",
st->module_name ? st->module_name : "",
st->units,
st->name,
rrdset_plugin_name(st),
rrdset_module_name(st),
rrdset_units(st),
rrdset_name(st),
rrdset_type_name(st->chart_type));
if (likely(!skip_volatile))
@ -98,9 +98,9 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
buffer_strcat(wb, ",\n\t\t\t\t\"");
else
buffer_strcat(wb, "\t\t\t\t\"");
buffer_strcat_jsonescape(wb, rd->id);
buffer_strcat_jsonescape(wb, rrddim_id(rd));
buffer_strcat(wb, "\": { \"name\": \"");
buffer_strcat_jsonescape(wb, rd->name);
buffer_strcat_jsonescape(wb, rrddim_name(rd));
buffer_strcat(wb, "\" }");
dimensions++;
@ -121,7 +121,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n");
size_t alarms = 0;
RRDCALC *rc;
for (rc = st->alarms; rc; rc = rc->rrdset_next) {
foreach_rrdcalc_in_rrdset(st, rc) {
buffer_sprintf(
wb,
"%s"
@ -131,7 +131,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
"\t\t\t\t\t\"units\": \"%s\",\n"
"\t\t\t\t\t\"update_every\": %d\n"
"\t\t\t\t}",
(alarms) ? ",\n" : "", rc->name, rc->id, rrdcalc_status2string(rc->status), rc->units,
(alarms) ? ",\n" : "", rrdcalc_name(rc), rc->id, rrdcalc_status2string(rc->status), rrdcalc_units(rc),
rc->update_every);
alarms++;

51
web/api/queries/query.c
View File

@ -665,11 +665,11 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options,
RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return;
if (should_lock)
rrdset_check_rdlock(r->st);
if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return;
int match_ids = 0, match_names = 0;
if(unlikely(options & RRDR_OPTION_MATCH_IDS))
@ -685,8 +685,8 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options,
RRDDIM *d;
long c, dims_selected = 0, dims_not_hidden_not_zero = 0;
for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next) {
if( (match_ids && simple_pattern_matches(pattern, d->id))
|| (match_names && simple_pattern_matches(pattern, d->name))
if( (match_ids && simple_pattern_matches(pattern, rrddim_id(d)))
|| (match_names && simple_pattern_matches(pattern, rrddim_name(d)))
) {
r->od[c] |= RRDR_DIMENSION_SELECTED;
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) r->od[c] &= ~RRDR_DIMENSION_HIDDEN;
@ -738,11 +738,11 @@ static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) {
internal_error(rrdr_line >= r->n,
"QUERY: requested to step above RRDR size for chart '%s'",
r->st->name);
rrdset_name(r->st));
internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t,
"QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'",
(size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name);
(size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, rrdset_name(r->st));
// save the time
r->t[rrdr_line] = t;
@ -768,7 +768,7 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted,
internal_error(true, "QUERY: NULL dimension - invalid params to tier calculation");
else
internal_error(true, "QUERY: chart '%s' dimension '%s' invalid params to tier calculation",
(rd->rrdset)?rd->rrdset->name:"unknown", rd->name);
(rd->rrdset)?rrdset_name(rd->rrdset):"unknown", rrddim_name(rd));
return 0;
}
@ -782,7 +782,7 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted,
for(int tier = 0; tier < storage_tiers ; tier++) {
if(unlikely(!rd->tiers[tier])) {
internal_error(true, "QUERY: tier %d of chart '%s' dimension '%s' not initialized",
tier, rd->rrdset->name, rd->name);
tier, rrdset_name(rd->rrdset), rrddim_name(rd));
// buffer_free(wb);
return 0;
}
@ -799,7 +799,7 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted,
int update_every = (int)rd->tiers[tier]->tier_grouping * (int)rd->update_every;
if(unlikely(update_every == 0)) {
internal_error(true, "QUERY: update_every of tier %d for chart '%s' dimension '%s' is zero. tg = %d, ue = %d",
tier, rd->rrdset->name, rd->name, rd->tiers[tier]->tier_grouping, rd->update_every);
tier, rrdset_name(rd->rrdset), rrddim_name(rd), rd->tiers[tier]->tier_grouping, rd->update_every);
// buffer_free(wb);
return 0;
}
@ -851,7 +851,7 @@ static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t aft
if(!st->dimensions->tiers[best_tier]) {
internal_error(
true,
"QUERY: tier %d on chart '%s', is not initialized", best_tier, st->name);
"QUERY: tier %d on chart '%s', is not initialized", best_tier, rrdset_name(st));
}
else {
ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every;
@ -859,7 +859,7 @@ static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t aft
internal_error(
true,
"QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d",
st->name, st->dimensions->tiers[best_tier]->tier_grouping, st->update_every);
rrdset_name(st), st->dimensions->tiers[best_tier]->tier_grouping, st->update_every);
ret = st->update_every;
}
@ -1259,7 +1259,7 @@ static inline void rrd2rrdr_do_dimension(
// check if the db is giving us zero duration points
if(unlikely(new_point.start_time == new_point.end_time)) {
internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal",
rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time);
rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time);
new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every);
}
@ -1267,7 +1267,7 @@ static inline void rrd2rrdr_do_dimension(
// check if the db is advancing the query
if(unlikely(new_point.end_time <= last1_point.end_time)) {
internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld",
rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time,
rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time,
last1_point.id, last1_point.end_time, now_start_time, now_end_time);
count_same_end_time++;
@ -1295,7 +1295,7 @@ static inline void rrd2rrdr_do_dimension(
// we only log if this is not point 1
internal_error(new_point.end_time < after_wanted && new_point.id > 1,
"QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)",
rd->rrdset->name, rd->name,
rrdset_name(rd->rrdset), rrddim_name(rd),
new_point.id, new_point.start_time, new_point.end_time,
now_start_time, now_end_time,
after_wanted, before_wanted);
@ -1339,7 +1339,7 @@ static inline void rrd2rrdr_do_dimension(
internal_error(current_point.id > 0 && last1_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,
"QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"
" interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point1 in this query.",
rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
rrdset_name(rd->rrdset), rrddim_name(rd), after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
current_point.id, current_point.start_time, current_point.end_time, now_end_time);
}
else if(likely(now_end_time <= last1_point.end_time)) {
@ -1350,7 +1350,7 @@ static inline void rrd2rrdr_do_dimension(
internal_error(current_point.id > 0 && last2_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,
"QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"
" interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point2 in this query.",
rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
rrdset_name(rd->rrdset), rrddim_name(rd), after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
current_point.id, current_point.start_time, current_point.end_time, now_end_time);
}
else {
@ -1425,7 +1425,7 @@ static inline void rrd2rrdr_do_dimension(
internal_error((long)points_added != points_wanted,
"QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).",
r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
rrdset_name(r->st), rrddim_name(rd), (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
}
// ----------------------------------------------------------------------------
@ -1518,7 +1518,7 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
//"slot (after: %zu, before: %zu, delta: %zu), "
"points (got: %ld, want: %ld, req: %ld, db: %ld), "
"%s"
, r->st->name
, rrdset_name(r->st)
, r->st->update_every
// grouping
@ -1733,7 +1733,7 @@ RRDR *rrd2rrdr(
rrdset_unlock(st);
if(first_entry_t == 0 || last_entry_t == 0) {
internal_error(true, "QUERY: chart without data detected on '%s'", st->name);
internal_error(true, "QUERY: chart without data detected on '%s'", rrdset_name(st));
query_debug_log_free();
return NULL;
}
@ -1932,13 +1932,13 @@ RRDR *rrd2rrdr(
RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list);
if(unlikely(!r)) {
internal_error(true, "QUERY: cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld",
st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
rrdset_id(st), (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
return NULL;
}
if(unlikely(!r->d || !points_wanted)) {
internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld",
st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);
rrdset_id(st), (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);
return r;
}
@ -1977,10 +1977,9 @@ RRDR *rrd2rrdr(
if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
rrdset_check_rdlock(st);
if(dimensions)
if(dimensions && *dimensions)
rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
query_debug_log_fin();
// -------------------------------------------------------------------------
@ -2024,21 +2023,21 @@ RRDR *rrd2rrdr(
else {
if(r->after != max_after) {
internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
st->name, (size_t)max_after, rd->name, (size_t)r->after);
rrdset_name(st), (size_t)max_after, rrddim_name(rd), (size_t)r->after);
r->after = (r->after > max_after) ? r->after : max_after;
}
if(r->before != min_before) {
internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
st->name, (size_t)min_before, rd->name, (size_t)r->before);
rrdset_name(st), (size_t)min_before, rrddim_name(rd), (size_t)r->before);
r->before = (r->before < min_before) ? r->before : min_before;
}
if(r->rows != max_rows) {
internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
st->name, (size_t)max_rows, rd->name, (size_t)r->rows);
rrdset_name(st), (size_t)max_rows, rrddim_name(rd), (size_t)r->rows);
r->rows = (r->rows > max_rows) ? r->rows : max_rows;
}

36
web/api/queries/weights.c
View File

@ -121,14 +121,14 @@ static void register_result(DICTIONARY *results,
struct register_result t = {
.flags = flags,
.st = st,
.chart_id = st->id,
.context = st->context,
.dim_name = d->name,
.chart_id = rrdset_id(st),
.context = rrdset_context(st),
.dim_name = rrddim_name(d),
.value = v
};
char buf[5000 + 1];
snprintfz(buf, 5000, "%s:%s", st->id, d->name);
snprintfz(buf, 5000, "%s:%s", rrdset_id(st), rrddim_name(d));
dictionary_set(results, buf, &t, sizeof(struct register_result));
}
@ -541,7 +541,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
group_time, options, NULL, context_param_list, group_options,
timeout, tier);
if(!high_rrdr) {
info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name);
info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", rrdset_name(st));
goto cleanup;
}
@ -551,11 +551,11 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
stats->db_points += high_rrdr->internal.db_points_read;
stats->result_points += high_rrdr->internal.result_points_generated;
if(!high_rrdr->d) {
info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name);
info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", rrdset_name(st));
goto cleanup;
}
if(high_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", st->name);
info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", rrdset_name(st));
goto cleanup;
}
int high_points = rrdr_rows(high_rrdr);
@ -571,7 +571,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
group_time, options, NULL, context_param_list, group_options,
(int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), tier);
if(!base_rrdr) {
info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name);
info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", rrdset_name(st));
goto cleanup;
}
@ -581,15 +581,15 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
stats->db_points += base_rrdr->internal.db_points_read;
stats->result_points += base_rrdr->internal.result_points_generated;
if(!base_rrdr->d) {
info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name);
info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", rrdset_name(st));
goto cleanup;
}
if (base_rrdr->d != high_rrdr->d) {
info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", st->name);
info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", rrdset_name(st));
goto cleanup;
}
if(base_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", st->name);
info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", rrdset_name(st));
goto cleanup;
}
int base_points = rrdr_rows(base_rrdr);
@ -605,7 +605,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
// for each dimension
RRDDIM *d;
int i;
for(i = 0, d = base_rrdr->st->dimensions ; d && i < base_rrdr->d; i++, d = d->next) {
for(i = 0, d = base_rrdr->st->dimensions; d && i < base_rrdr->d; i++, d = d->next) {
// skip the not evaluated ones
if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN))
@ -692,7 +692,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
NETDATA_DOUBLE baseline_average = NAN;
NETDATA_DOUBLE base_anomaly_rate = 0;
value_is_null = 1;
ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1,
ret = rrdset2value_api_v1(st, NULL, &baseline_average, rrddim_id(d), 1,
baseline_after, baseline_before,
group, group_options, group_time, options,
NULL, NULL,
@ -709,7 +709,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
NETDATA_DOUBLE highlight_average = NAN;
NETDATA_DOUBLE high_anomaly_rate = 0;
value_is_null = 1;
ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1,
ret = rrdset2value_api_v1(st, NULL, &highlight_average, rrddim_id(d), 1,
after, before,
group, group_options, group_time, options,
NULL, NULL,
@ -734,7 +734,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
char highlighted_countif_options[50 + 1];
snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average);
ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1,
ret = rrdset2value_api_v1(st, NULL, &highlight_countif, rrddim_id(d), 1,
after, before,
RRDR_GROUPING_COUNTIF,highlighted_countif_options,
group_time, options,
@ -803,7 +803,7 @@ static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results,
NETDATA_DOUBLE average = NAN;
NETDATA_DOUBLE anomaly_rate = 0;
value_is_null = 1;
ret = rrdset2value_api_v1(st, NULL, &average, d->id, 1,
ret = rrdset2value_api_v1(st, NULL, &average, rrddim_id(d), 1,
after, before,
group, group_options, group_time, options,
NULL, NULL,
@ -1006,8 +1006,8 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
rrdhost_rdlock(host);
rrdset_foreach_read(st, host) {
if (rrdset_is_available_for_viewers(st)) {
if(!contexts || simple_pattern_matches(contexts, st->context))
dictionary_set(charts, st->name, NULL, 0);
if(!contexts || simple_pattern_matches(contexts, rrdset_context(st)))
dictionary_set(charts, rrdset_name(st), NULL, 0);
}
}
rrdhost_unlock(host);

14
web/api/web_api_v1.c
View File

@ -701,8 +701,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
if (context && !chart) {
RRDSET *st1;
uint32_t context_hash = simple_hash(context);
SIMPLE_PATTERN *chart_label_key_pattern = NULL;
if(chart_label_key)
chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
@ -711,14 +709,16 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
if(chart_labels_filter)
chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
STRING *context_string = string_strdupz(context);
rrdhost_rdlock(host);
rrdset_foreach_read(st1, host) {
if (st1->hash_context == context_hash && !strcmp(st1->context, context) &&
if (st1->context == context_string &&
(!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_label_key_pattern, ':')) &&
(!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_labels_filter_pattern, ':')))
build_context_param_list(owa, &context_param_list, st1);
}
rrdhost_unlock(host);
string_freez(context_string);
if (likely(context_param_list && context_param_list->rd)) // Just set the first one
st = context_param_list->rd->rrdset;
@ -1055,7 +1055,7 @@ static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST
int alarm_normal = 0, alarm_warn = 0, alarm_crit = 0;
RRDCALC *rc;
rrdhost_rdlock(host);
for(rc = host->alarms; rc ; rc = rc->next) {
foreach_rrdcalc_in_rrdhost(host, rc) {
if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
continue;
@ -1086,7 +1086,7 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {
if (count > 0)
buffer_strcat(wb, ",\n");
buffer_sprintf(wb, "\t\t\"%s\"", host->hostname);
buffer_sprintf(wb, "\t\t\"%s\"", rrdhost_hostname(host));
count++;
}
@ -1101,7 +1101,7 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {
buffer_sprintf(
wb, "\t\t{ \"guid\": \"%s\", \"hostname\": \"%s\", \"reachable\": %s, \"hops\": %d"
, host->machine_guid
, host->hostname
, rrdhost_hostname(host)
, (host->receiver || host == localhost) ? "true" : "false"
, host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1
);
@ -1148,7 +1148,7 @@ extern int aclk_connected;
inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
{
buffer_strcat(wb, "{\n");
buffer_sprintf(wb, "\t\"version\": \"%s\",\n", host->program_version);
buffer_sprintf(wb, "\t\"version\": \"%s\",\n", rrdhost_program_version(host));
buffer_sprintf(wb, "\t\"uid\": \"%s\",\n", host->machine_guid);
web_client_api_request_v1_info_mirrored_hosts(wb);

6
web/server/web_client.c
View File

@ -1305,11 +1305,9 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
if(url && *url) strncpyz(&w->last_url[1], url, NETDATA_WEB_REQUEST_URL_SIZE - 1);
else w->last_url[1] = '\0';
uint32_t hash = simple_hash(tok);
host = rrdhost_find_by_hostname(tok, hash);
host = rrdhost_find_by_hostname(tok);
if (!host)
host = rrdhost_find_by_guid(tok, hash);
host = rrdhost_find_by_guid(tok);
if (!host) {
host = sql_create_host_by_uuid(tok);
if (likely(host)) {