diff --git a/CMakeLists.txt b/CMakeLists.txt index c2dfcc02ef..02fc8304e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -220,7 +220,11 @@ pkg_check_modules(CURL libcurl) # ----------------------------------------------------------------------------- # Detect libcups -pkg_check_modules(CURL libcups) +pkg_check_modules(CUPS libcups) +IF(NOT CUPS_LIBRARIES) + pkg_check_modules(CUPS cups) +ENDIF() + # later we use: # ${CUPS_LIBRARIES} # ${CUPS_CFLAGS_OTHER} @@ -446,6 +450,8 @@ set(LIBNETDATA_FILES libnetdata/avl/avl.h libnetdata/buffer/buffer.c libnetdata/buffer/buffer.h + libnetdata/circular_buffer/circular_buffer.c + libnetdata/circular_buffer/circular_buffer.h libnetdata/clocks/clocks.c libnetdata/clocks/clocks.h libnetdata/completion/completion.c @@ -454,10 +460,15 @@ set(LIBNETDATA_FILES libnetdata/dictionary/dictionary.h libnetdata/eval/eval.c libnetdata/eval/eval.h + libnetdata/health/health.c + libnetdata/health/health.h libnetdata/inlined.h + libnetdata/json/json.c + libnetdata/json/json.h + libnetdata/json/jsmn.c + libnetdata/json/jsmn.h libnetdata/libnetdata.c libnetdata/libnetdata.h - libnetdata/required_dummies.h libnetdata/locks/locks.c libnetdata/locks/locks.h libnetdata/log/log.c @@ -470,6 +481,9 @@ set(LIBNETDATA_FILES libnetdata/popen/popen.h libnetdata/procfile/procfile.c libnetdata/procfile/procfile.h + libnetdata/required_dummies.h + libnetdata/socket/security.c + libnetdata/socket/security.h libnetdata/simple_pattern/simple_pattern.c libnetdata/simple_pattern/simple_pattern.h libnetdata/socket/socket.c @@ -478,23 +492,16 @@ set(LIBNETDATA_FILES libnetdata/statistical/statistical.h libnetdata/storage_number/storage_number.c libnetdata/storage_number/storage_number.h + libnetdata/string/string.c + libnetdata/string/string.h libnetdata/threads/threads.c libnetdata/threads/threads.h libnetdata/url/url.c libnetdata/url/url.h - libnetdata/json/json.c - libnetdata/json/json.h - libnetdata/json/jsmn.c - libnetdata/json/jsmn.h - libnetdata/health/health.c - libnetdata/health/health.h libnetdata/string/utf8.h - libnetdata/socket/security.c - libnetdata/socket/security.h libnetdata/worker_utilization/worker_utilization.c libnetdata/worker_utilization/worker_utilization.h - libnetdata/circular_buffer/circular_buffer.c - libnetdata/circular_buffer/circular_buffer.h) + ) IF(ENABLE_PLUGIN_EBPF) list(APPEND LIBNETDATA_FILES diff --git a/Makefile.am b/Makefile.am index e56a91f4fe..df535e1c77 100644 --- a/Makefile.am +++ b/Makefile.am @@ -173,6 +173,8 @@ LIBNETDATA_FILES = \ libnetdata/socket/security.h \ libnetdata/statistical/statistical.c \ libnetdata/statistical/statistical.h \ + libnetdata/string/string.c \ + libnetdata/string/string.h \ libnetdata/storage_number/storage_number.c \ libnetdata/storage_number/storage_number.h \ libnetdata/threads/threads.c \ diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c index b214b3731b..74aedf2b52 100644 --- a/collectors/cgroups.plugin/sys_fs_cgroup.c +++ b/collectors/cgroups.plugin/sys_fs_cgroup.c @@ -857,16 +857,16 @@ struct cgroup { char *filename_cpu_cfs_quota; unsigned long long cpu_cfs_quota; - RRDSETVAR *chart_var_cpu_limit; + const RRDSETVAR_ACQUIRED *chart_var_cpu_limit; NETDATA_DOUBLE prev_cpu_usage; char *filename_memory_limit; unsigned long long memory_limit; - RRDSETVAR *chart_var_memory_limit; + const RRDSETVAR_ACQUIRED *chart_var_memory_limit; char *filename_memoryswap_limit; unsigned long long memoryswap_limit; - RRDSETVAR *chart_var_memoryswap_limit; + const RRDSETVAR_ACQUIRED *chart_var_memoryswap_limit; // services RRDDIM *rd_cpu; @@ -3708,10 +3708,10 @@ cpu_limits2_err: } } -static inline int update_memory_limits(char **filename, RRDSETVAR **chart_var, unsigned long long *value, const char *chart_var_name, struct cgroup *cg) { +static inline int update_memory_limits(char **filename, const RRDSETVAR_ACQUIRED **chart_var, unsigned long long *value, const char *chart_var_name, struct cgroup *cg) { if(*filename) { if(unlikely(!*chart_var)) { - *chart_var = rrdsetvar_custom_chart_variable_create(cg->st_mem_usage, chart_var_name); + *chart_var = rrdsetvar_custom_chart_variable_add_and_acquire(cg->st_mem_usage, chart_var_name); if(!*chart_var) { error("Cannot create cgroup %s chart variable '%s'. Will not update its limit anymore.", cg->id, chart_var_name); freez(*filename); @@ -3727,7 +3727,7 @@ static inline int update_memory_limits(char **filename, RRDSETVAR **chart_var, u *filename = NULL; } else { - rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024))); + rrdsetvar_custom_chart_variable_set(cg->st_mem_usage, *chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024))); return 1; } } else { @@ -3742,11 +3742,11 @@ static inline int update_memory_limits(char **filename, RRDSETVAR **chart_var, u char *s = "max\n\0"; if(strcmp(s, buffer) == 0){ *value = UINT64_MAX; - rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024))); + rrdsetvar_custom_chart_variable_set(cg->st_mem_usage, *chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024))); return 1; } *value = str2ull(buffer); - rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024))); + rrdsetvar_custom_chart_variable_set(cg->st_mem_usage, *chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024))); return 1; } } @@ -3843,7 +3843,7 @@ void update_cgroup_charts(int update_every) { } if(unlikely(!cg->chart_var_cpu_limit)) { - cg->chart_var_cpu_limit = rrdsetvar_custom_chart_variable_create(cg->st_cpu, "cpu_limit"); + cg->chart_var_cpu_limit = rrdsetvar_custom_chart_variable_add_and_acquire(cg->st_cpu, "cpu_limit"); if(!cg->chart_var_cpu_limit) { error("Cannot create cgroup %s chart variable 'cpu_limit'. Will not update its limit anymore.", cg->id); if(cg->filename_cpuset_cpus) freez(cg->filename_cpuset_cpus); @@ -3868,8 +3868,6 @@ void update_cgroup_charts(int update_every) { value = (NETDATA_DOUBLE)cg->cpuset_cpus * 100; } if(likely(value)) { - rrdsetvar_custom_chart_variable_set(cg->chart_var_cpu_limit, value); - if(unlikely(!cg->st_cpu_limit)) { snprintfz(title, CHART_TITLE_MAX, "CPU Usage within the limits"); @@ -3909,14 +3907,15 @@ void update_cgroup_charts(int update_every) { cg->prev_cpu_usage = cpu_usage; + rrdsetvar_custom_chart_variable_set(cg->st_cpu, cg->chart_var_cpu_limit, value); rrdset_done(cg->st_cpu_limit); } else { - rrdsetvar_custom_chart_variable_set(cg->chart_var_cpu_limit, NAN); if(unlikely(cg->st_cpu_limit)) { rrdset_is_obsolete(cg->st_cpu_limit); cg->st_cpu_limit = NULL; } + rrdsetvar_custom_chart_variable_set(cg->st_cpu, cg->chart_var_cpu_limit, NAN); } } } diff --git a/collectors/cgroups.plugin/tests/test_doubles.c b/collectors/cgroups.plugin/tests/test_doubles.c index 7f44416255..9aa748725c 100644 --- a/collectors/cgroups.plugin/tests/test_doubles.c +++ b/collectors/cgroups.plugin/tests/test_doubles.c @@ -146,14 +146,6 @@ void netdev_rename_device_del(const char *host_device) UNUSED(host_device); } -void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value) -{ - UNUSED(chart_uuid); - UNUSED(source_type); - UNUSED(label); - UNUSED(value); -} - void rrdcalc_update_rrdlabels(RRDSET *st) { (void)st; } diff --git a/collectors/cups.plugin/cups_plugin.c b/collectors/cups.plugin/cups_plugin.c index 77bd3e0ed8..9a200c31d1 100644 --- a/collectors/cups.plugin/cups_plugin.c +++ b/collectors/cups.plugin/cups_plugin.c @@ -137,10 +137,7 @@ getIntegerOption( return ((int)intvalue); } -static int reset_job_metrics(const char *name, void *entry, void *data) { - (void)name; - (void)data; - +static int reset_job_metrics(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data __maybe_unused) { struct job_metrics *jm = (struct job_metrics *)entry; jm->is_collected = 0; @@ -175,8 +172,8 @@ struct job_metrics *get_job_metrics(char *dest) { return jm; } -int collect_job_metrics(const char *name, void *entry, void *data) { - (void)data; +int collect_job_metrics(const DICTIONARY_ITEM *item, void *entry, void *data __maybe_unused) { + const char *name = dictionary_acquired_item_name(item); struct job_metrics *jm = (struct job_metrics *)entry; @@ -205,7 +202,7 @@ int collect_job_metrics(const char *name, void *entry, void *data) { printf("DIMENSION pending '' absolute 1 1\n"); printf("DIMENSION held '' absolute 1 1\n"); printf("DIMENSION processing '' absolute 1 1\n"); - dictionary_del_having_write_lock(dict_dest_job_metrics, name); + dictionary_del(dict_dest_job_metrics, name); } return 0; @@ -243,7 +240,7 @@ int main(int argc, char **argv) { errno = 0; - dict_dest_job_metrics = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict_dest_job_metrics = dictionary_create(DICT_OPTION_SINGLE_THREADED); // ------------------------------------------------------------------------ // the main loop diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c index 5bdf8bc61f..7381e1ec93 100644 --- a/collectors/diskspace.plugin/plugin_diskspace.c +++ b/collectors/diskspace.plugin/plugin_diskspace.c @@ -95,8 +95,8 @@ int mount_point_cleanup(const char *name, void *entry, int slow) { return 0; } -int mount_point_cleanup_cb(const char *name, void *entry, void *data) { - UNUSED(data); +int mount_point_cleanup_cb(const DICTIONARY_ITEM *item, void *entry, void *data __maybe_unused) { + const char *name = dictionary_acquired_item_name(item); return mount_point_cleanup(name, (struct mount_point_metadata *)entry, 0); } @@ -330,7 +330,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { , SIMPLE_PATTERN_EXACT ); - dict_mountpoints = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict_mountpoints = dictionary_create(DICT_OPTION_SINGLE_THREADED); } struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point); diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index 3dffde5acc..ce9250d568 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -101,15 +101,19 @@ PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char * UNUSED(user); if (global) { - RRDVAR *rv = rrdvar_custom_host_variable_create(host, name); - if (rv) - rrdvar_custom_host_variable_set(host, rv, value); + const RRDVAR_ACQUIRED *rva = rrdvar_custom_host_variable_add_and_acquire(host, name); + if (rva) { + rrdvar_custom_host_variable_set(host, rva, value); + rrdvar_custom_host_variable_release(host, rva); + } else error("cannot find/create HOST VARIABLE '%s' on host '%s'", name, rrdhost_hostname(host)); } else { - RRDSETVAR *rs = rrdsetvar_custom_chart_variable_create(st, name); - if (rs) - rrdsetvar_custom_chart_variable_set(rs, value); + const RRDSETVAR_ACQUIRED *rsa = rrdsetvar_custom_chart_variable_add_and_acquire(st, name); + if (rsa) { + rrdsetvar_custom_chart_variable_set(st, rsa, value); + rrdsetvar_custom_chart_variable_release(st, rsa); + } else error("cannot find/create CHART VARIABLE '%s' on host '%s', chart '%s'", name, rrdhost_hostname(host), rrdset_id(st)); } @@ -127,7 +131,7 @@ PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name RRDDIM *rd = rrddim_add(st, id, name, multiplier, divisor, algorithm_type); int unhide_dimension = 1; - rrddim_flag_clear(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); + rrddim_option_clear(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS); if (options && *options) { if (strstr(options, "obsolete") != NULL) rrddim_is_obsolete(st, rd); @@ -137,20 +141,20 @@ PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name unhide_dimension = !strstr(options, "hidden"); if (strstr(options, "noreset") != NULL) - rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); + rrddim_option_set(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS); if (strstr(options, "nooverflow") != NULL) - rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); + rrddim_option_set(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS); } else rrddim_isnot_obsolete(st, rd); if (likely(unhide_dimension)) { - rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN); + rrddim_option_clear(rd, RRDDIM_OPTION_HIDDEN); if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) { (void)sql_set_dimension_option(&rd->metric_uuid, NULL); rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN); } } else { - rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN); + rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN); if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) { (void)sql_set_dimension_option(&rd->metric_uuid, "hidden"); rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN); diff --git a/collectors/proc.plugin/ipc.c b/collectors/proc.plugin/ipc.c index e114a05a41..e0863f6d8b 100644 --- a/collectors/proc.plugin/ipc.c +++ b/collectors/proc.plugin/ipc.c @@ -281,7 +281,7 @@ int do_ipc(int update_every, usec_t dt) { static int read_limits_next = -1; static struct ipc_limits limits; static struct ipc_status status; - static RRDVAR *arrays_max = NULL, *semaphores_max = NULL; + static const RRDVAR_ACQUIRED *arrays_max = NULL, *semaphores_max = NULL; static RRDSET *st_semaphores = NULL, *st_arrays = NULL; static RRDDIM *rd_semaphores = NULL, *rd_arrays = NULL; static char *msg_filename = NULL; @@ -352,8 +352,8 @@ int do_ipc(int update_every, usec_t dt) { } // variables - semaphores_max = rrdvar_custom_host_variable_create(localhost, "ipc_semaphores_max"); - arrays_max = rrdvar_custom_host_variable_create(localhost, "ipc_semaphores_arrays_max"); + semaphores_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_max"); + arrays_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_arrays_max"); } struct stat stbuf; @@ -483,11 +483,7 @@ int do_ipc(int update_every, usec_t dt) { rrdset_done(st_msq_messages); rrdset_done(st_msq_bytes); - long long dimensions_num = 0; - RRDDIM *rd; - rrdset_rdlock(st_msq_messages); - rrddim_foreach_read(rd, st_msq_messages) dimensions_num++; - rrdset_unlock(st_msq_messages); + long long dimensions_num = rrdset_number_of_dimensions(st_msq_messages); if(unlikely(dimensions_num > dimensions_limit)) { info("Message queue statistics has been disabled"); diff --git a/collectors/proc.plugin/proc_diskstats.c b/collectors/proc.plugin/proc_diskstats.c index 945ca25216..554bc4f897 100644 --- a/collectors/proc.plugin/proc_diskstats.c +++ b/collectors/proc.plugin/proc_diskstats.c @@ -874,8 +874,6 @@ static void add_labels_to_disk(struct disk *d, RRDSET *st) { rrdlabels_add(st->rrdlabels, "device_type", "virtual", RRDLABEL_SRC_AUTO); break; } - - rrdcalc_update_rrdlabels(st); } int do_proc_diskstats(int update_every, usec_t dt) { diff --git a/collectors/proc.plugin/proc_interrupts.c b/collectors/proc.plugin/proc_interrupts.c index db2a20b5df..17c287425d 100644 --- a/collectors/proc.plugin/proc_interrupts.c +++ b/collectors/proc.plugin/proc_interrupts.c @@ -175,7 +175,7 @@ int do_proc_interrupts(int update_every, usec_t dt) { // calls of this function. if(unlikely(!irr->rd || strncmp(rrddim_name(irr->rd), irr->name, MAX_INTERRUPT_NAME) != 0)) { irr->rd = rrddim_add(st_system_interrupts, irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_set_name(st_system_interrupts, irr->rd, irr->name); + rrddim_reset_name(st_system_interrupts, irr->rd, irr->name); // also reset per cpu RRDDIMs to avoid repeating strncmp() in the per core loop if(likely(do_per_core != CONFIG_BOOLEAN_NO)) { @@ -237,7 +237,7 @@ int do_proc_interrupts(int update_every, usec_t dt) { if(irr->used && (do_per_core == CONFIG_BOOLEAN_YES || irr->cpu[c].value)) { if(unlikely(!irr->cpu[c].rd)) { irr->cpu[c].rd = rrddim_add(core_st[c], irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_set_name(core_st[c], irr->cpu[c].rd, irr->name); + rrddim_reset_name(core_st[c], irr->cpu[c].rd, irr->name); } rrddim_set_by_pointer(core_st[c], irr->cpu[c].rd, irr->cpu[c].value); diff --git a/collectors/proc.plugin/proc_loadavg.c b/collectors/proc.plugin/proc_loadavg.c index 8b78ecc9ef..14415ef65b 100644 --- a/collectors/proc.plugin/proc_loadavg.c +++ b/collectors/proc.plugin/proc_loadavg.c @@ -99,7 +99,7 @@ int do_proc_loadavg(int update_every, usec_t dt) { if(likely(do_all_processes)) { static RRDSET *processes_chart = NULL; static RRDDIM *rd_active = NULL; - static RRDSETVAR *rd_pidmax; + static const RRDSETVAR_ACQUIRED *rd_pidmax; if(unlikely(!processes_chart)) { processes_chart = rrdset_create_localhost( @@ -118,12 +118,12 @@ int do_proc_loadavg(int update_every, usec_t dt) { ); rd_active = rrddim_add(processes_chart, "active", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_pidmax = rrdsetvar_custom_chart_variable_create(processes_chart, "pidmax"); + rd_pidmax = rrdsetvar_custom_chart_variable_add_and_acquire(processes_chart, "pidmax"); } else rrdset_next(processes_chart); rrddim_set_by_pointer(processes_chart, rd_active, active_processes); - rrdsetvar_custom_chart_variable_set(rd_pidmax, max_processes); + rrdsetvar_custom_chart_variable_set(processes_chart, rd_pidmax, max_processes); rrdset_done(processes_chart); } diff --git a/collectors/proc.plugin/proc_mdstat.c b/collectors/proc.plugin/proc_mdstat.c index 77b75d3139..2bfb6913f3 100644 --- a/collectors/proc.plugin/proc_mdstat.c +++ b/collectors/proc.plugin/proc_mdstat.c @@ -80,7 +80,6 @@ static inline void make_chart_obsolete(char *name, const char *id_modifier) static void add_labels_to_mdstat(struct raid *raid, RRDSET *st) { rrdlabels_add(st->rrdlabels, "device", raid->name, RRDLABEL_SRC_AUTO); rrdlabels_add(st->rrdlabels, "raid_level", raid->level, RRDLABEL_SRC_AUTO); - rrdcalc_update_rrdlabels(st); } int do_proc_mdstat(int update_every, usec_t dt) diff --git a/collectors/proc.plugin/proc_net_dev.c b/collectors/proc.plugin/proc_net_dev.c index 79572f4420..701cc00ea7 100644 --- a/collectors/proc.plugin/proc_net_dev.c +++ b/collectors/proc.plugin/proc_net_dev.c @@ -188,7 +188,7 @@ static struct netdev { RRDDIM *rd_mtu; char *filename_speed; - RRDSETVAR *chart_var_speed; + const RRDSETVAR_ACQUIRED *chart_var_speed; char *filename_duplex; char *filename_operstate; @@ -967,7 +967,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { // update the interface speed if(d->filename_speed) { if(unlikely(!d->chart_var_speed)) { - d->chart_var_speed = rrdsetvar_custom_chart_variable_create(d->st_bandwidth, "nic_speed_max"); + d->chart_var_speed = + rrdsetvar_custom_chart_variable_add_and_acquire(d->st_bandwidth, "nic_speed_max"); if(!d->chart_var_speed) { error("Cannot create interface %s chart variable 'nic_speed_max'. Will not update its speed anymore.", d->name); freez(d->filename_speed); @@ -990,8 +991,6 @@ int do_proc_net_dev(int update_every, usec_t dt) { d->filename_speed = NULL; } else { - rrdsetvar_custom_chart_variable_set(d->chart_var_speed, (NETDATA_DOUBLE) d->speed * KILOBITS_IN_A_MEGABIT); - if(d->do_speed != CONFIG_BOOLEAN_NO) { if(unlikely(!d->st_speed)) { d->st_speed = rrdset_create_localhost( @@ -1020,6 +1019,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrddim_set_by_pointer(d->st_speed, d->rd_speed, (collected_number)d->speed * KILOBITS_IN_A_MEGABIT); rrdset_done(d->st_speed); } + + rrdsetvar_custom_chart_variable_set(d->st_bandwidth, d->chart_var_speed, (NETDATA_DOUBLE) d->speed * KILOBITS_IN_A_MEGABIT); } } } diff --git a/collectors/proc.plugin/proc_net_snmp.c b/collectors/proc.plugin/proc_net_snmp.c index b03a6ac74d..d667a06df8 100644 --- a/collectors/proc.plugin/proc_net_snmp.c +++ b/collectors/proc.plugin/proc_net_snmp.c @@ -104,7 +104,7 @@ int do_proc_net_snmp(int update_every, usec_t dt) { *arl_udp = NULL, *arl_udplite = NULL; - static RRDVAR *tcp_max_connections_var = NULL; + static const RRDVAR_ACQUIRED *tcp_max_connections_var = NULL; if(unlikely(!arl_ip)) { do_ip_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 packets", CONFIG_BOOLEAN_AUTO); @@ -216,7 +216,7 @@ int do_proc_net_snmp(int update_every, usec_t dt) { arl_expect(arl_udplite, "InCsumErrors", &snmp_root.udplite_InCsumErrors); arl_expect(arl_udplite, "IgnoredMulti", &snmp_root.udplite_IgnoredMulti); - tcp_max_connections_var = rrdvar_custom_host_variable_create(localhost, "tcp_max_connections"); + tcp_max_connections_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_connections"); } if(unlikely(!ff)) { diff --git a/collectors/proc.plugin/proc_net_sockstat.c b/collectors/proc.plugin/proc_net_sockstat.c index 994cbad7b6..db75bee1ab 100644 --- a/collectors/proc.plugin/proc_net_sockstat.c +++ b/collectors/proc.plugin/proc_net_sockstat.c @@ -27,14 +27,14 @@ static struct proc_net_sockstat { static int read_tcp_mem(void) { static char *filename = NULL; - static RRDVAR *tcp_mem_low_threshold = NULL, + static const RRDVAR_ACQUIRED *tcp_mem_low_threshold = NULL, *tcp_mem_pressure_threshold = NULL, *tcp_mem_high_threshold = NULL; if(unlikely(!tcp_mem_low_threshold)) { - tcp_mem_low_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_low"); - tcp_mem_pressure_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_pressure"); - tcp_mem_high_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_high"); + tcp_mem_low_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_low"); + tcp_mem_pressure_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_pressure"); + tcp_mem_high_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_high"); } if(unlikely(!filename)) { @@ -69,7 +69,7 @@ static int read_tcp_mem(void) { static kernel_uint_t read_tcp_max_orphans(void) { static char *filename = NULL; - static RRDVAR *tcp_max_orphans_var = NULL; + static const RRDVAR_ACQUIRED *tcp_max_orphans_var = NULL; if(unlikely(!filename)) { char buffer[FILENAME_MAX + 1]; @@ -81,7 +81,7 @@ static kernel_uint_t read_tcp_max_orphans(void) { if(read_single_number_file(filename, &tcp_max_orphans) == 0) { if(unlikely(!tcp_max_orphans_var)) - tcp_max_orphans_var = rrdvar_custom_host_variable_create(localhost, "tcp_max_orphans"); + tcp_max_orphans_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_orphans"); rrdvar_custom_host_variable_set(localhost, tcp_max_orphans_var, tcp_max_orphans); return tcp_max_orphans; diff --git a/collectors/proc.plugin/proc_net_stat_conntrack.c b/collectors/proc.plugin/proc_net_stat_conntrack.c index 642e33f8e9..3182a745a6 100644 --- a/collectors/proc.plugin/proc_net_stat_conntrack.c +++ b/collectors/proc.plugin/proc_net_stat_conntrack.c @@ -12,7 +12,7 @@ int do_proc_net_stat_conntrack(int update_every, usec_t dt) { static usec_t get_max_every = 10 * USEC_PER_SEC, usec_since_last_max = 0; static int read_full = 1; static char *nf_conntrack_filename, *nf_conntrack_count_filename, *nf_conntrack_max_filename; - static RRDVAR *rrdvar_max = NULL; + static const RRDVAR_ACQUIRED *rrdvar_max = NULL; unsigned long long aentries = 0, asearched = 0, afound = 0, anew = 0, ainvalid = 0, aignore = 0, adelete = 0, adelete_list = 0, ainsert = 0, ainsert_failed = 0, adrop = 0, aearly_drop = 0, aicmp_error = 0, aexpect_new = 0, aexpect_create = 0, aexpect_delete = 0, asearch_restart = 0; @@ -50,7 +50,7 @@ int do_proc_net_stat_conntrack(int update_every, usec_t dt) { if(!do_sockets && !read_full) return 1; - rrdvar_max = rrdvar_custom_host_variable_create(localhost, "netfilter_conntrack_max"); + rrdvar_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "netfilter_conntrack_max"); } if(likely(read_full)) { diff --git a/collectors/proc.plugin/proc_net_wireless.c b/collectors/proc.plugin/proc_net_wireless.c index 328ab1ca05..78d8ebbac1 100644 --- a/collectors/proc.plugin/proc_net_wireless.c +++ b/collectors/proc.plugin/proc_net_wireless.c @@ -200,7 +200,6 @@ static void configure_device(int do_status, int do_quality, int do_discarded_pac static void add_labels_to_wireless(struct netwireless *w, RRDSET *st) { rrdlabels_add(st->rrdlabels, "device", w->name, RRDLABEL_SRC_AUTO); - rrdcalc_update_rrdlabels(st); } int do_proc_net_wireless(int update_every, usec_t dt) diff --git a/collectors/proc.plugin/proc_self_mountinfo.c b/collectors/proc.plugin/proc_self_mountinfo.c index 4456d5978a..9310f2ffc1 100644 --- a/collectors/proc.plugin/proc_self_mountinfo.c +++ b/collectors/proc.plugin/proc_self_mountinfo.c @@ -227,7 +227,8 @@ struct mountinfo *mountinfo_read(int do_statvfs) { struct mountinfo *root = NULL, *last = NULL, *mi = NULL; // create a dictionary to track uniqueness - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE); + DICTIONARY *dict = dictionary_create( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_NAME_LINK_DONT_CLONE); unsigned long l, lines = procfile_lines(ff); for(l = 0; l < lines ;l++) { diff --git a/collectors/proc.plugin/proc_softirqs.c b/collectors/proc.plugin/proc_softirqs.c index d6389161a2..a1c8b8ee73 100644 --- a/collectors/proc.plugin/proc_softirqs.c +++ b/collectors/proc.plugin/proc_softirqs.c @@ -155,7 +155,7 @@ int do_proc_softirqs(int update_every, usec_t dt) { // calls of this function. if(unlikely(!irr->rd || strncmp(irr->name, rrddim_name(irr->rd), MAX_INTERRUPT_NAME) != 0)) { irr->rd = rrddim_add(st_system_softirqs, irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_set_name(st_system_softirqs, irr->rd, irr->name); + rrddim_reset_name(st_system_softirqs, irr->rd, irr->name); // also reset per cpu RRDDIMs to avoid repeating strncmp() in the per core loop if(likely(do_per_core != CONFIG_BOOLEAN_NO)) { @@ -231,7 +231,7 @@ int do_proc_softirqs(int update_every, usec_t dt) { if(irr->used && (do_per_core == CONFIG_BOOLEAN_YES || irr->cpu[c].value)) { if(unlikely(!irr->cpu[c].rd)) { irr->cpu[c].rd = rrddim_add(core_st[c], irr->id, irr->name, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_set_name(core_st[c], irr->cpu[c].rd, irr->name); + rrddim_reset_name(core_st[c], irr->cpu[c].rd, irr->name); } rrddim_set_by_pointer(core_st[c], irr->cpu[c].rd, irr->cpu[c].value); diff --git a/collectors/proc.plugin/proc_spl_kstat_zfs.c b/collectors/proc.plugin/proc_spl_kstat_zfs.c index fae112249d..1b9ee04c10 100644 --- a/collectors/proc.plugin/proc_spl_kstat_zfs.c +++ b/collectors/proc.plugin/proc_spl_kstat_zfs.c @@ -252,8 +252,8 @@ void disable_zfs_pool_state(struct zfs_pool *pool) pool->disabled = 1; } -int update_zfs_pool_state_chart(const char *name, void *pool_p, void *update_every_p) -{ +int update_zfs_pool_state_chart(const DICTIONARY_ITEM *item, void *pool_p, void *update_every_p) { + const char *name = dictionary_acquired_item_name(item); struct zfs_pool *pool = (struct zfs_pool *)pool_p; int update_every = *(int *)update_every_p; @@ -321,7 +321,7 @@ int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt) snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/spl/kstat/zfs"); dirname = config_get("plugin:proc:" ZFS_PROC_POOLS, "directory to monitor", filename); - zfs_pools = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + zfs_pools = dictionary_create(DICT_OPTION_SINGLE_THREADED); do_zfs_pool_state = 1; } diff --git a/collectors/proc.plugin/proc_stat.c b/collectors/proc.plugin/proc_stat.c index 5086ff2c7d..dcc67e9ab6 100644 --- a/collectors/proc.plugin/proc_stat.c +++ b/collectors/proc.plugin/proc_stat.c @@ -481,7 +481,7 @@ int do_proc_stat(int update_every, usec_t dt) { static uint32_t hash_intr, hash_ctxt, hash_processes, hash_procs_running, hash_procs_blocked; static char *core_throttle_count_filename = NULL, *package_throttle_count_filename = NULL, *scaling_cur_freq_filename = NULL, *time_in_state_filename = NULL, *schedstat_filename = NULL, *cpuidle_name_filename = NULL, *cpuidle_time_filename = NULL; - static RRDVAR *cpus_var = NULL; + static const RRDVAR_ACQUIRED *cpus_var = NULL; static int accurate_freq_avail = 0, accurate_freq_is_used = 0; size_t cores_found = (size_t)processors; @@ -713,7 +713,7 @@ int do_proc_stat(int update_every, usec_t dt) { rrddim_hide(cpu_chart->st, "idle"); if(unlikely(core == 0 && cpus_var == NULL)) - cpus_var = rrdvar_custom_host_variable_create(localhost, "active_processors"); + cpus_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "active_processors"); } else rrdset_next(cpu_chart->st); diff --git a/collectors/proc.plugin/sys_block_zram.c b/collectors/proc.plugin/sys_block_zram.c index 42c467630c..55136f502f 100644 --- a/collectors/proc.plugin/sys_block_zram.c +++ b/collectors/proc.plugin/sys_block_zram.c @@ -177,7 +177,7 @@ static void free_device(DICTIONARY *dict, const char *name) rrdset_obsolete_and_pointer_null(d->st_savings); rrdset_obsolete_and_pointer_null(d->st_alloc_efficiency); rrdset_obsolete_and_pointer_null(d->st_comp_ratio); - dictionary_del_having_write_lock(dict, name); + dictionary_del(dict, name); } // -------------------------------------------------------------------- @@ -239,13 +239,16 @@ static inline int _collect_zram_metrics(const char* name, ZRAM_DEVICE *d, int ad return 0; } -static int collect_first_zram_metrics(const char *name, void *entry, void *data) { +static int collect_first_zram_metrics(const DICTIONARY_ITEM *item, void *entry, void *data) { + const char *name = dictionary_acquired_item_name(item); + // collect without calling rrdset_next (init only) return _collect_zram_metrics(name, (ZRAM_DEVICE *)entry, 0, (DICTIONARY *)data); } -static int collect_zram_metrics(const char *name, void *entry, void *data) { - (void)name; +static int collect_zram_metrics(const DICTIONARY_ITEM *item, void *entry, void *data) { + const char *name = dictionary_acquired_item_name(item); + // collect with calling rrdset_next return _collect_zram_metrics(name, (ZRAM_DEVICE *)entry, 1, (DICTIONARY *)data); } @@ -280,7 +283,7 @@ int do_sys_block_zram(int update_every, usec_t dt) { } procfile_close(ff); - devices = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + devices = dictionary_create(DICT_OPTION_SINGLE_THREADED); device_count = init_devices(devices, (unsigned int)zram_id, update_every); if (device_count < 1) return 1; diff --git a/collectors/proc.plugin/sys_class_infiniband.c b/collectors/proc.plugin/sys_class_infiniband.c index 7e63bcbb4e..603f2eaa2c 100644 --- a/collectors/proc.plugin/sys_class_infiniband.c +++ b/collectors/proc.plugin/sys_class_infiniband.c @@ -184,7 +184,7 @@ static struct ibport { RRDSET *st_hwpackets; RRDSET *st_hwerrors; - RRDSETVAR *stv_speed; + const RRDSETVAR_ACQUIRED *stv_speed; usec_t speed_last_collected_usec; @@ -543,7 +543,7 @@ int do_sys_class_infiniband(int update_every, usec_t dt) // x4 lanes multiplier as per Documentation/ABI/stable/sysfs-class-infiniband FOREACH_COUNTER_BYTES(GEN_RRD_DIM_ADD_CUSTOM, port, 4 * 8 * port->width, 1024, RRD_ALGORITHM_INCREMENTAL) - port->stv_speed = rrdsetvar_custom_chart_variable_create(port->st_bytes, "link_speed"); + port->stv_speed = rrdsetvar_custom_chart_variable_add_and_acquire(port->st_bytes, "link_speed"); } else rrdset_next(port->st_bytes); @@ -551,7 +551,7 @@ int do_sys_class_infiniband(int update_every, usec_t dt) FOREACH_COUNTER_BYTES(GEN_RRD_DIM_SETP, port) // For link speed set only variable - rrdsetvar_custom_chart_variable_set(port->stv_speed, port->speed); + rrdsetvar_custom_chart_variable_set(port->st_bytes, port->stv_speed, port->speed); rrdset_done(port->st_bytes); } diff --git a/collectors/proc.plugin/sys_class_power_supply.c b/collectors/proc.plugin/sys_class_power_supply.c index 40731b5d5f..a8f9ac4462 100644 --- a/collectors/proc.plugin/sys_class_power_supply.c +++ b/collectors/proc.plugin/sys_class_power_supply.c @@ -114,7 +114,6 @@ void power_supply_free(struct power_supply *ps) { static void add_labels_to_power_supply(struct power_supply *ps, RRDSET *st) { rrdlabels_add(st->rrdlabels, "device", ps->name, RRDLABEL_SRC_AUTO); - rrdcalc_update_rrdlabels(st); } int do_sys_class_power_supply(int update_every, usec_t dt) { diff --git a/collectors/proc.plugin/sys_fs_btrfs.c b/collectors/proc.plugin/sys_fs_btrfs.c index 323bb69671..c43632f8ad 100644 --- a/collectors/proc.plugin/sys_fs_btrfs.c +++ b/collectors/proc.plugin/sys_fs_btrfs.c @@ -451,7 +451,6 @@ static inline int find_all_btrfs_pools(const char *path) { static void add_labels_to_btrfs(BTRFS_NODE *n, RRDSET *st) { rrdlabels_add(st->rrdlabels, "device", n->id, RRDLABEL_SRC_AUTO); rrdlabels_add(st->rrdlabels, "device_label", n->label, RRDLABEL_SRC_AUTO); - rrdcalc_update_rrdlabels(st); } int do_sys_fs_btrfs(int update_every, usec_t dt) { diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c index e68640a4f7..84facd8085 100644 --- a/collectors/statsd.plugin/statsd.c +++ b/collectors/statsd.plugin/statsd.c @@ -24,9 +24,9 @@ #ifdef STATSD_MULTITHREADED // DO NOT ENABLE MULTITHREADING - IT IS NOT WELL TESTED -#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DONT_OVERWRITE_VALUE|DICTIONARY_FLAG_ADD_IN_FRONT +#define STATSD_DICTIONARY_OPTIONS (DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_ADD_IN_FRONT) #else -#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DONT_OVERWRITE_VALUE|DICTIONARY_FLAG_ADD_IN_FRONT|DICTIONARY_FLAG_SINGLE_THREADED +#define STATSD_DICTIONARY_OPTIONS (DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_ADD_IN_FRONT | DICT_OPTION_SINGLE_THREADED) #endif #define STATSD_DECIMAL_DETAIL 1000 // floating point values get multiplied by this, with the same divisor @@ -192,6 +192,7 @@ typedef struct statsd_app_chart_dimension { collected_number multiplier; // the multiplier of the dimension collected_number divisor; // the divisor of the dimension RRDDIM_FLAGS flags; // the RRDDIM flags for this dimension + RRDDIM_OPTIONS options; // the RRDDIM options for this dimension STATSD_APP_CHART_DIM_VALUE_TYPE value_type; // which value to use of the source metric @@ -371,9 +372,10 @@ static struct statsd { // -------------------------------------------------------------------------------------------------------------------- // statsd index management - add/find metrics -static void dictionary_metric_insert_callback(const char *name, void *value, void *data) { +static void dictionary_metric_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) { STATSD_INDEX *index = (STATSD_INDEX *)data; STATSD_METRIC *m = (STATSD_METRIC *)value; + const char *name = dictionary_acquired_item_name(item); debug(D_STATSD, "Creating new %s metric '%s'", index->name, name); @@ -390,9 +392,9 @@ static void dictionary_metric_insert_callback(const char *name, void *value, voi __atomic_fetch_add(&index->metrics, 1, __ATOMIC_RELAXED); } -static void dictionary_metric_delete_callback(const char *name, void *value, void *data) { +static void dictionary_metric_delete_callback(const DICTIONARY_ITEM *item, void *value, void *data) { (void)data; // STATSD_INDEX *index = (STATSD_INDEX *)data; - (void)name; + (void)item; STATSD_METRIC *m = (STATSD_METRIC *)value; if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) { @@ -416,7 +418,7 @@ static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, cons // no locks here, go faster // this will call the dictionary_metric_insert_callback() if an item // is inserted, otherwise it will return the existing one. - // We used the flag DICTIONARY_FLAG_DONT_OVERWRITE_VALUE to support this. + // We used the flag DICT_OPTION_DONT_OVERWRITE_VALUE to support this. STATSD_METRIC *m = dictionary_set(index->dict, name, NULL, sizeof(STATSD_METRIC)); #endif @@ -572,8 +574,8 @@ static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const cha #define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer") #define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram") -static void dictionary_metric_set_value_insert_callback(const char *name, void *value, void *data) { - (void)name; +static void dictionary_metric_set_value_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + (void)item; (void)value; STATSD_METRIC *m = (STATSD_METRIC *)data; m->set.unique++; @@ -617,8 +619,8 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) { } } -static void dictionary_metric_dict_value_insert_callback(const char *name, void *value, void *data) { - (void)name; +static void dictionary_metric_dict_value_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + (void)item; (void)value; STATSD_METRIC *m = (STATSD_METRIC *)data; m->dictionary.unique++; @@ -1209,6 +1211,7 @@ static STATSD_APP_CHART_DIM *add_dimension_to_app_chart( , collected_number multiplier , collected_number divisor , RRDDIM_FLAGS flags + , RRDDIM_OPTIONS options , STATSD_APP_CHART_DIM_VALUE_TYPE value_type ) { STATSD_APP_CHART_DIM *dim = callocz(sizeof(STATSD_APP_CHART_DIM), 1); @@ -1221,6 +1224,7 @@ static STATSD_APP_CHART_DIM *add_dimension_to_app_chart( dim->divisor = divisor; dim->value_type = value_type; dim->flags = flags; + dim->options = options; if(!dim->multiplier) dim->multiplier = 1; @@ -1323,7 +1327,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA else if(app) { if(!strcmp(s, "dictionary")) { if(!app->dict) - app->dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + app->dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); dict = app->dict; } @@ -1474,17 +1478,18 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA pattern = 1; } - char *dim_name = words[i++]; - char *type = words[i++]; - char *multiplier = words[i++]; - char *divisor = words[i++]; - char *options = words[i++]; + char *dim_name = words[i++]; + char *type = words[i++]; + char *multiplier = words[i++]; + char *divisor = words[i++]; + char *opts = words[i++]; RRDDIM_FLAGS flags = RRDDIM_FLAG_NONE; - if(options && *options) { - if(strstr(options, "hidden") != NULL) flags |= RRDDIM_FLAG_HIDDEN; - if(strstr(options, "noreset") != NULL) flags |= RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS; - if(strstr(options, "nooverflow") != NULL) flags |= RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS; + RRDDIM_OPTIONS options = RRDDIM_OPTION_NONE; + if(opts && *opts) { + if(strstr(opts, "hidden") != NULL) options |= RRDDIM_OPTION_HIDDEN; + if(strstr(opts, "noreset") != NULL) options |= RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS; + if(strstr(opts, "nooverflow") != NULL) options |= RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS; } if(!pattern) { @@ -1510,7 +1515,8 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA , (multiplier && *multiplier)?str2l(multiplier):1 , (divisor && *divisor)?str2l(divisor):1 , flags - , string2valuetype(type, line, filename) + , + options, string2valuetype(type, line, filename) ); if(pattern) @@ -1775,7 +1781,7 @@ static inline void statsd_private_chart_dictionary(STATSD_METRIC *m) { STATSD_METRIC_DICTIONARY_ITEM *t; dfe_start_read(m->dictionary.dict, t) { - if (!t->rd) t->rd = rrddim_add(m->st, t_name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + if (!t->rd) t->rd = rrddim_add(m->st, t_dfe.name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_set_by_pointer(m->st, t->rd, (collected_number)t->count); } dfe_done(t); @@ -2130,6 +2136,7 @@ static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC , dim->multiplier , dim->divisor , dim->flags + , dim->options , dim->value_type ); @@ -2186,11 +2193,13 @@ static inline RRDDIM *statsd_add_dim_to_app_chart(STATSD_APP *app, STATSD_APP_CH dim->rd = rrddim_add(chart->st, metric, dim->name, dim->multiplier, dim->divisor, dim->algorithm); if(dim->flags != RRDDIM_FLAG_NONE) dim->rd->flags |= dim->flags; + if(dim->options != RRDDIM_OPTION_NONE) dim->rd->options |= dim->options; return dim->rd; } dim->rd = rrddim_add(chart->st, dim->metric, dim->name, dim->multiplier, dim->divisor, dim->algorithm); if(dim->flags != RRDDIM_FLAG_NONE) dim->rd->flags |= dim->flags; + if(dim->options != RRDDIM_OPTION_NONE) dim->rd->options |= dim->options; return dim->rd; } diff --git a/collectors/tc.plugin/plugin_tc.c b/collectors/tc.plugin/plugin_tc.c index f4ef659435..393a9575d2 100644 --- a/collectors/tc.plugin/plugin_tc.c +++ b/collectors/tc.plugin/plugin_tc.c @@ -74,7 +74,7 @@ struct tc_device { // ---------------------------------------------------------------------------- // tc_class index -static void tc_class_free_callback(const char *name __maybe_unused, void *value, void *data __maybe_unused) { +static void tc_class_free_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { // struct tc_device *d = data; struct tc_class *c = value; @@ -84,22 +84,21 @@ static void tc_class_free_callback(const char *name __maybe_unused, void *value, string_freez(c->parentid); } -static void tc_class_conflict_callback(const char *name __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { +static bool tc_class_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { struct tc_device *d = data; (void)d; struct tc_class *c = old_value; (void)c; struct tc_class *new_c = new_value; (void)new_c; - error("TC: class '%s' is already in device '%s'. Ignoring duplicate.", name, string2str(d->id)); + error("TC: class '%s' is already in device '%s'. Ignoring duplicate.", dictionary_acquired_item_name(item), string2str(d->id)); - tc_class_free_callback(name, new_value, data); + tc_class_free_callback(item, new_value, data); + + return true; } static void tc_class_index_init(struct tc_device *d) { if(!d->classes) { - d->classes = dictionary_create( - DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - |DICTIONARY_FLAG_SINGLE_THREADED - ); + d->classes = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED); dictionary_register_delete_callback(d->classes, tc_class_free_callback, d); dictionary_register_conflict_callback(d->classes, tc_class_conflict_callback, d); @@ -128,12 +127,12 @@ static inline struct tc_class *tc_class_index_find(struct tc_device *d, const ch static DICTIONARY *tc_device_root_index = NULL; -static void tc_device_add_callback(const char *name __maybe_unused, void *value, void *data __maybe_unused) { +static void tc_device_add_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct tc_device *d = value; tc_class_index_init(d); } -static void tc_device_free_callback(const char *name __maybe_unused, void *value, void *data __maybe_unused) { +static void tc_device_free_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct tc_device *d = value; tc_class_index_destroy(d); @@ -146,10 +145,7 @@ static void tc_device_free_callback(const char *name __maybe_unused, void *value static void tc_device_index_init() { if(!tc_device_root_index) { tc_device_root_index = dictionary_create( - DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - |DICTIONARY_FLAG_SINGLE_THREADED - |DICTIONARY_FLAG_ADD_IN_FRONT - ); + DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED | DICT_OPTION_ADD_IN_FRONT); dictionary_register_insert_callback(tc_device_root_index, tc_device_add_callback, NULL); dictionary_register_delete_callback(tc_device_root_index, tc_device_free_callback, NULL); @@ -196,7 +192,7 @@ static inline void tc_device_classes_cleanup(struct tc_device *d) { d->family_updated = false; struct tc_class *c; - dfe_start_unsafe(d->classes, c) { + dfe_start_write(d->classes, c) { if(unlikely(cleanup_every && c->unupdated >= cleanup_every)) tc_class_free(d, c); @@ -254,7 +250,7 @@ static inline void tc_device_commit(struct tc_device *d) { // prepare all classes // we set reasonable defaults for the rest of the code below - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { c->render = false; // do not render this class c->isleaf = true; // this is a leaf class c->hasparent = false; // without a parent @@ -283,7 +279,7 @@ static inline void tc_device_commit(struct tc_device *d) { error("TC: device '%s' has active both classes (%d) and qdiscs (%d). Will render only qdiscs.", string2str(d->id), updated_classes, updated_qdiscs); // set all classes to !updated - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if (unlikely(!c->isqdisc && c->updated)) c->updated = false; } @@ -307,7 +303,7 @@ static inline void tc_device_commit(struct tc_device *d) { // so, here we remove the isleaf flag from nodes in the middle // and we add the hasparent flag to leaf nodes we found their parent if(likely(!d->enabled_all_classes_qdiscs)) { - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if(unlikely(!c->updated)) continue; @@ -319,7 +315,7 @@ static inline void tc_device_commit(struct tc_device *d) { // c->parentid?c->parentid:"NULL"); // find if c is leaf or not - dfe_start_unsafe(d->classes, x) { + dfe_start_read(d->classes, x) { if(unlikely(!x->updated || c == x || !x->parentid)) continue; @@ -339,7 +335,7 @@ static inline void tc_device_commit(struct tc_device *d) { dfe_done(c); } - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if(unlikely(!c->updated)) continue; @@ -367,7 +363,7 @@ static inline void tc_device_commit(struct tc_device *d) { // dump all the list to see what we know if(unlikely(debug_flags & D_TC_LOOP)) { - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if(c->render) debug(D_TC_LOOP, "TC: final nodes dump for '%s': class %s, OK", string2str(d->name), string2str(c->id)); else debug(D_TC_LOOP, "TC: final nodes dump for '%s': class '%s', IGNORE (updated: %d, isleaf: %d, hasparent: %d, parent: '%s')", string2str(d->name?d->name:d->id), string2str(c->id), c->updated, c->isleaf, c->hasparent, string2str(c->parentid)); @@ -426,7 +422,8 @@ static inline void tc_device_commit(struct tc_device *d) { } else { rrdset_next(d->st_bytes); - if(unlikely(d->name_updated)) rrdset_set_name(d->st_bytes, string2str(d->name)); + if(unlikely(d->name_updated)) + rrdset_reset_name(d->st_bytes, string2str(d->name)); if(d->name && d->name_updated) rrdlabels_add(d->st_bytes->rrdlabels, "name", string2str(d->name), RRDLABEL_SRC_AUTO); @@ -438,13 +435,13 @@ static inline void tc_device_commit(struct tc_device *d) { // update the family } - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if(unlikely(!c->render)) continue; if(unlikely(!c->rd_bytes)) c->rd_bytes = rrddim_add(d->st_bytes, string2str(c->id), string2str(c->name?c->name:c->id), 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); else if(unlikely(c->name_updated)) - rrddim_set_name(d->st_bytes, c->rd_bytes, string2str(c->name)); + rrddim_reset_name(d->st_bytes, c->rd_bytes, string2str(c->name)); rrddim_set_by_pointer(d->st_bytes, c->rd_bytes, c->bytes); } @@ -491,7 +488,7 @@ static inline void tc_device_commit(struct tc_device *d) { if(unlikely(d->name_updated)) { char name[RRD_ID_LENGTH_MAX + 1]; snprintfz(name, RRD_ID_LENGTH_MAX, "%s_packets", string2str(d->name?d->name:d->id)); - rrdset_set_name(d->st_packets, name); + rrdset_reset_name(d->st_packets, name); } if(d->name && d->name_updated) @@ -504,13 +501,13 @@ static inline void tc_device_commit(struct tc_device *d) { // update the family } - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if(unlikely(!c->render)) continue; if(unlikely(!c->rd_packets)) c->rd_packets = rrddim_add(d->st_packets, string2str(c->id), string2str(c->name?c->name:c->id), 1, 1, RRD_ALGORITHM_INCREMENTAL); else if(unlikely(c->name_updated)) - rrddim_set_name(d->st_packets, c->rd_packets, string2str(c->name)); + rrddim_reset_name(d->st_packets, c->rd_packets, string2str(c->name)); rrddim_set_by_pointer(d->st_packets, c->rd_packets, c->packets); } @@ -557,7 +554,7 @@ static inline void tc_device_commit(struct tc_device *d) { if(unlikely(d->name_updated)) { char name[RRD_ID_LENGTH_MAX + 1]; snprintfz(name, RRD_ID_LENGTH_MAX, "%s_dropped", string2str(d->name?d->name:d->id)); - rrdset_set_name(d->st_dropped, name); + rrdset_reset_name(d->st_dropped, name); } if(d->name && d->name_updated) @@ -570,13 +567,13 @@ static inline void tc_device_commit(struct tc_device *d) { // update the family } - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if(unlikely(!c->render)) continue; if(unlikely(!c->rd_dropped)) c->rd_dropped = rrddim_add(d->st_dropped, string2str(c->id), string2str(c->name?c->name:c->id), 1, 1, RRD_ALGORITHM_INCREMENTAL); else if(unlikely(c->name_updated)) - rrddim_set_name(d->st_dropped, c->rd_dropped, string2str(c->name)); + rrddim_reset_name(d->st_dropped, c->rd_dropped, string2str(c->name)); rrddim_set_by_pointer(d->st_dropped, c->rd_dropped, c->dropped); } @@ -623,7 +620,7 @@ static inline void tc_device_commit(struct tc_device *d) { if(unlikely(d->name_updated)) { char name[RRD_ID_LENGTH_MAX + 1]; snprintfz(name, RRD_ID_LENGTH_MAX, "%s_tokens", string2str(d->name?d->name:d->id)); - rrdset_set_name(d->st_tokens, name); + rrdset_reset_name(d->st_tokens, name); } if(d->name && d->name_updated) @@ -636,14 +633,14 @@ static inline void tc_device_commit(struct tc_device *d) { // update the family } - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if(unlikely(!c->render)) continue; if(unlikely(!c->rd_tokens)) { c->rd_tokens = rrddim_add(d->st_tokens, string2str(c->id), string2str(c->name?c->name:c->id), 1, 1, RRD_ALGORITHM_ABSOLUTE); } else if(unlikely(c->name_updated)) - rrddim_set_name(d->st_tokens, c->rd_tokens, string2str(c->name)); + rrddim_reset_name(d->st_tokens, c->rd_tokens, string2str(c->name)); rrddim_set_by_pointer(d->st_tokens, c->rd_tokens, c->tokens); } @@ -691,7 +688,7 @@ static inline void tc_device_commit(struct tc_device *d) { if(unlikely(d->name_updated)) { char name[RRD_ID_LENGTH_MAX + 1]; snprintfz(name, RRD_ID_LENGTH_MAX, "%s_ctokens", string2str(d->name?d->name:d->id)); - rrdset_set_name(d->st_ctokens, name); + rrdset_reset_name(d->st_ctokens, name); } if(d->name && d->name_updated) @@ -704,13 +701,13 @@ static inline void tc_device_commit(struct tc_device *d) { // update the family } - dfe_start_unsafe(d->classes, c) { + dfe_start_read(d->classes, c) { if(unlikely(!c->render)) continue; if(unlikely(!c->rd_ctokens)) c->rd_ctokens = rrddim_add(d->st_ctokens, string2str(c->id), string2str(c->name?c->name:c->id), 1, 1, RRD_ALGORITHM_ABSOLUTE); else if(unlikely(c->name_updated)) - rrddim_set_name(d->st_ctokens, c->rd_ctokens, string2str(c->name)); + rrddim_reset_name(d->st_ctokens, c->rd_ctokens, string2str(c->name)); rrddim_set_by_pointer(d->st_ctokens, c->rd_ctokens, c->ctokens); } @@ -1146,12 +1143,12 @@ void *tc_main(void *ptr) { worker_is_busy(WORKER_TC_WORKTIME); worker_set_metric(WORKER_TC_PLUGIN_TIME, str2ll(words[1], NULL)); - size_t number_of_devices = dictionary_stats_entries(tc_device_root_index); + size_t number_of_devices = dictionary_entries(tc_device_root_index); size_t number_of_classes = 0; struct tc_device *d; - dfe_start_unsafe(tc_device_root_index, d) { - number_of_classes += dictionary_stats_entries(d->classes); + dfe_start_read(tc_device_root_index, d) { + number_of_classes += dictionary_entries(d->classes); } dfe_done(d); diff --git a/configure.ac b/configure.ac index 40cd3869da..d7a3b4509f 100644 --- a/configure.ac +++ b/configure.ac @@ -1718,6 +1718,7 @@ AC_CONFIG_FILES([ libnetdata/simple_pattern/Makefile libnetdata/socket/Makefile libnetdata/statistical/Makefile + libnetdata/string/Makefile libnetdata/storage_number/Makefile libnetdata/storage_number/tests/Makefile libnetdata/threads/Makefile diff --git a/daemon/analytics.c b/daemon/analytics.c index 6c40930b75..bf8668d065 100644 --- a/daemon/analytics.c +++ b/daemon/analytics.c @@ -249,8 +249,7 @@ void analytics_exporters(void) buffer_free(bi); } -int collector_counter_callb(const char *name, void *entry, void *data) { - (void)name; +int collector_counter_callb(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { struct array_printer *ap = (struct array_printer *)data; struct collector *col = (struct collector *)entry; @@ -279,21 +278,22 @@ int collector_counter_callb(const char *name, void *entry, void *data) { void analytics_collectors(void) { RRDSET *st; - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); char name[500]; BUFFER *bt = buffer_create(1000); - rrdset_foreach_read(st, localhost) - { - if (rrdset_is_available_for_viewers(st)) { - struct collector col = { - .plugin = rrdset_plugin_name(st), - .module = rrdset_module_name(st) - }; - snprintfz(name, 499, "%s:%s", col.plugin, col.module); - dictionary_set(dict, name, &col, sizeof(struct collector)); - } + rrdset_foreach_read(st, localhost) { + if(!rrdset_is_available_for_viewers(st)) + continue; + + struct collector col = { + .plugin = rrdset_plugin_name(st), + .module = rrdset_module_name(st) + }; + snprintfz(name, 499, "%s:%s", col.plugin, col.module); + dictionary_set(dict, name, &col, sizeof(struct collector)); } + rrdset_foreach_done(st); struct array_printer ap; ap.c = 0; @@ -398,12 +398,11 @@ void analytics_charts(void) { RRDSET *st; int c = 0; + rrdset_foreach_read(st, localhost) - { - if (rrdset_is_available_for_viewers(st)) { - c++; - } - } + if(rrdset_is_available_for_viewers(st)) c++; + rrdset_foreach_done(st); + { char b[7]; snprintfz(b, 6, "%d", c); @@ -415,22 +414,19 @@ void analytics_metrics(void) { RRDSET *st; long int dimensions = 0; - RRDDIM *rd; - rrdset_foreach_read(st, localhost) - { - rrdset_rdlock(st); - + rrdset_foreach_read(st, localhost) { if (rrdset_is_available_for_viewers(st)) { - rrddim_foreach_read(rd, st) - { - if (rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; dimensions++; } + rrddim_foreach_done(rd); } - - rrdset_unlock(st); } + rrdset_foreach_done(st); + { char b[7]; snprintfz(b, 6, "%ld", dimensions); @@ -443,7 +439,7 @@ void analytics_alarms(void) int alarm_warn = 0, alarm_crit = 0, alarm_normal = 0; char b[10]; RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(localhost, rc) { + foreach_rrdcalc_in_rrdhost_read(localhost, rc) { if (unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; @@ -458,6 +454,7 @@ void analytics_alarms(void) alarm_normal++; } } + foreach_rrdcalc_in_rrdhost_done(rc); snprintfz(b, 9, "%d", alarm_normal); analytics_set_data(&analytics_data.netdata_alarms_normal, b); @@ -527,16 +524,11 @@ void analytics_gather_immutable_meta_data(void) */ void analytics_gather_mutable_meta_data(void) { - rrdhost_rdlock(localhost); - analytics_collectors(); analytics_alarms(); analytics_charts(); analytics_metrics(); analytics_aclk(); - - rrdhost_unlock(localhost); - analytics_mirrored_hosts(); analytics_alarms_notifications(); diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index ce2cdc6a10..1d4da897a5 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -10,8 +10,9 @@ #define WORKER_JOB_DBENGINE 3 #define WORKER_JOB_HEARTBEAT 4 #define WORKER_JOB_STRINGS 5 +#define WORKER_JOB_DICTIONARIES 6 -#if WORKER_UTILIZATION_MAX_JOB_TYPES < 6 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 7 #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5 #endif @@ -1215,6 +1216,367 @@ static void update_heartbeat_charts() { rrdset_done(st_heartbeat); } +// --------------------------------------------------------------------------------------------------------------------- +// dictionary statistics + +struct dictionary_categories { + struct dictionary_stats *stats; + const char *family; + const char *context_prefix; + int priority; + + RRDSET *st_dicts; + RRDDIM *rd_dicts_active; + RRDDIM *rd_dicts_deleted; + + RRDSET *st_items; + RRDDIM *rd_items_entries; + RRDDIM *rd_items_referenced; + RRDDIM *rd_items_pending_deletion; + + RRDSET *st_ops; + RRDDIM *rd_ops_creations; + RRDDIM *rd_ops_destructions; + RRDDIM *rd_ops_flushes; + RRDDIM *rd_ops_traversals; + RRDDIM *rd_ops_walkthroughs; + RRDDIM *rd_ops_garbage_collections; + RRDDIM *rd_ops_searches; + RRDDIM *rd_ops_inserts; + RRDDIM *rd_ops_resets; + RRDDIM *rd_ops_deletes; + + RRDSET *st_callbacks; + RRDDIM *rd_callbacks_inserts; + RRDDIM *rd_callbacks_conflicts; + RRDDIM *rd_callbacks_reacts; + RRDDIM *rd_callbacks_deletes; + + RRDSET *st_memory; + RRDDIM *rd_memory_indexed; + RRDDIM *rd_memory_values; + RRDDIM *rd_memory_dict; + + RRDSET *st_spins; + RRDDIM *rd_spins_use; + RRDDIM *rd_spins_search; + RRDDIM *rd_spins_insert; + +} dictionary_categories[] = { + { .stats = &dictionary_stats_category_other, "dictionaries", "dictionaries", 900000 }, + + // terminator + { .stats = NULL, NULL, NULL, 0 }, +}; + +#define load_dictionary_stats_entry(x) total += (size_t)(stats.x = __atomic_load_n(&c->stats->x, __ATOMIC_RELAXED)) + +static void update_dictionary_category_charts(struct dictionary_categories *c) { + struct dictionary_stats stats; + stats.name = c->stats->name; + + // ------------------------------------------------------------------------ + + size_t total = 0; + load_dictionary_stats_entry(dictionaries.active); + load_dictionary_stats_entry(dictionaries.deleted); + + if(c->st_dicts || total != 0) { + if (unlikely(!c->st_dicts)) { + char id[RRD_ID_LENGTH_MAX + 1]; + snprintfz(id, RRD_ID_LENGTH_MAX, "%s.%s.dictionaries", c->context_prefix, stats.name); + + char context[RRD_ID_LENGTH_MAX + 1]; + snprintfz(context, RRD_ID_LENGTH_MAX, "netdata.%s.category.dictionaries", c->context_prefix); + + c->st_dicts = rrdset_create_localhost( + "netdata" + , id + , NULL + , c->family + , context + , "Dictionaries" + , "dictionaries" + , "netdata" + , "stats" + , c->priority + 0 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + c->rd_dicts_active = rrddim_add(c->st_dicts, "active", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + c->rd_dicts_deleted = rrddim_add(c->st_dicts, "deleted", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); + + rrdlabels_add(c->st_dicts->rrdlabels, "category", stats.name, RRDLABEL_SRC_AUTO); + } + else + rrdset_next(c->st_dicts); + + rrddim_set_by_pointer(c->st_dicts, c->rd_dicts_active, (collected_number)stats.dictionaries.active); + rrddim_set_by_pointer(c->st_dicts, c->rd_dicts_deleted, (collected_number)stats.dictionaries.deleted); + rrdset_done(c->st_dicts); + } + + // ------------------------------------------------------------------------ + + total = 0; + load_dictionary_stats_entry(items.entries); + load_dictionary_stats_entry(items.referenced); + load_dictionary_stats_entry(items.pending_deletion); + + if(c->st_items || total != 0) { + if (unlikely(!c->st_items)) { + char id[RRD_ID_LENGTH_MAX + 1]; + snprintfz(id, RRD_ID_LENGTH_MAX, "%s.%s.items", c->context_prefix, stats.name); + + char context[RRD_ID_LENGTH_MAX + 1]; + snprintfz(context, RRD_ID_LENGTH_MAX, "netdata.%s.category.items", c->context_prefix); + + c->st_items = rrdset_create_localhost( + "netdata" + , id + , NULL + , c->family + , context + , "Dictionary Items" + , "items" + , "netdata" + , "stats" + , c->priority + 1 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + c->rd_items_entries = rrddim_add(c->st_items, "active", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + c->rd_items_pending_deletion = rrddim_add(c->st_items, "deleted", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); + c->rd_items_referenced = rrddim_add(c->st_items, "referenced", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + rrdlabels_add(c->st_items->rrdlabels, "category", stats.name, RRDLABEL_SRC_AUTO); + } + else + rrdset_next(c->st_items); + + rrddim_set_by_pointer(c->st_items, c->rd_items_entries, stats.items.entries); + rrddim_set_by_pointer(c->st_items, c->rd_items_pending_deletion, stats.items.pending_deletion); + rrddim_set_by_pointer(c->st_items, c->rd_items_referenced, stats.items.referenced); + rrdset_done(c->st_items); + } + + // ------------------------------------------------------------------------ + + total = 0; + load_dictionary_stats_entry(ops.creations); + load_dictionary_stats_entry(ops.destructions); + load_dictionary_stats_entry(ops.flushes); + load_dictionary_stats_entry(ops.traversals); + load_dictionary_stats_entry(ops.walkthroughs); + load_dictionary_stats_entry(ops.garbage_collections); + load_dictionary_stats_entry(ops.searches); + load_dictionary_stats_entry(ops.inserts); + load_dictionary_stats_entry(ops.resets); + load_dictionary_stats_entry(ops.deletes); + + if(c->st_ops || total != 0) { + if (unlikely(!c->st_ops)) { + char id[RRD_ID_LENGTH_MAX + 1]; + snprintfz(id, RRD_ID_LENGTH_MAX, "%s.%s.ops", c->context_prefix, stats.name); + + char context[RRD_ID_LENGTH_MAX + 1]; + snprintfz(context, RRD_ID_LENGTH_MAX, "netdata.%s.category.ops", c->context_prefix); + + c->st_ops = rrdset_create_localhost( + "netdata" + , id + , NULL + , c->family + , context + , "Dictionary Operations" + , "ops/s" + , "netdata" + , "stats" + , c->priority + 2 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + c->rd_ops_creations = rrddim_add(c->st_ops, "creations", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_destructions = rrddim_add(c->st_ops, "destructions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_flushes = rrddim_add(c->st_ops, "flushes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_traversals = rrddim_add(c->st_ops, "traversals", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_walkthroughs = rrddim_add(c->st_ops, "walkthroughs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_garbage_collections = rrddim_add(c->st_ops, "garbage_collections", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_searches = rrddim_add(c->st_ops, "searches", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_inserts = rrddim_add(c->st_ops, "inserts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_resets = rrddim_add(c->st_ops, "resets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_ops_deletes = rrddim_add(c->st_ops, "deletes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrdlabels_add(c->st_ops->rrdlabels, "category", stats.name, RRDLABEL_SRC_AUTO); + } + else + rrdset_next(c->st_ops); + + rrddim_set_by_pointer(c->st_ops, c->rd_ops_creations, (collected_number)stats.ops.creations); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_destructions, (collected_number)stats.ops.destructions); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_flushes, (collected_number)stats.ops.flushes); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_traversals, (collected_number)stats.ops.traversals); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_walkthroughs, (collected_number)stats.ops.walkthroughs); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_garbage_collections, (collected_number)stats.ops.garbage_collections); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_searches, (collected_number)stats.ops.searches); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_inserts, (collected_number)stats.ops.inserts); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_resets, (collected_number)stats.ops.resets); + rrddim_set_by_pointer(c->st_ops, c->rd_ops_deletes, (collected_number)stats.ops.deletes); + + rrdset_done(c->st_ops); + } + + // ------------------------------------------------------------------------ + + total = 0; + load_dictionary_stats_entry(callbacks.inserts); + load_dictionary_stats_entry(callbacks.conflicts); + load_dictionary_stats_entry(callbacks.reacts); + load_dictionary_stats_entry(callbacks.deletes); + + if(c->st_callbacks || total != 0) { + if (unlikely(!c->st_callbacks)) { + char id[RRD_ID_LENGTH_MAX + 1]; + snprintfz(id, RRD_ID_LENGTH_MAX, "%s.%s.callbacks", c->context_prefix, stats.name); + + char context[RRD_ID_LENGTH_MAX + 1]; + snprintfz(context, RRD_ID_LENGTH_MAX, "netdata.%s.category.callbacks", c->context_prefix); + + c->st_callbacks = rrdset_create_localhost( + "netdata" + , id + , NULL + , c->family + , context + , "Dictionary Callbacks" + , "callbacks/s" + , "netdata" + , "stats" + , c->priority + 3 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + c->rd_callbacks_inserts = rrddim_add(c->st_callbacks, "inserts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_callbacks_deletes = rrddim_add(c->st_callbacks, "deletes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_callbacks_conflicts = rrddim_add(c->st_callbacks, "conflicts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_callbacks_reacts = rrddim_add(c->st_callbacks, "reacts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrdlabels_add(c->st_callbacks->rrdlabels, "category", stats.name, RRDLABEL_SRC_AUTO); + } + else + rrdset_next(c->st_callbacks); + + rrddim_set_by_pointer(c->st_callbacks, c->rd_callbacks_inserts, (collected_number)stats.callbacks.inserts); + rrddim_set_by_pointer(c->st_callbacks, c->rd_callbacks_conflicts, (collected_number)stats.callbacks.conflicts); + rrddim_set_by_pointer(c->st_callbacks, c->rd_callbacks_reacts, (collected_number)stats.callbacks.reacts); + rrddim_set_by_pointer(c->st_callbacks, c->rd_callbacks_deletes, (collected_number)stats.callbacks.deletes); + + rrdset_done(c->st_callbacks); + } + + // ------------------------------------------------------------------------ + + total = 0; + load_dictionary_stats_entry(memory.indexed); + load_dictionary_stats_entry(memory.values); + load_dictionary_stats_entry(memory.dict); + + if(c->st_memory || total != 0) { + if (unlikely(!c->st_memory)) { + char id[RRD_ID_LENGTH_MAX + 1]; + snprintfz(id, RRD_ID_LENGTH_MAX, "%s.%s.memory", c->context_prefix, stats.name); + + char context[RRD_ID_LENGTH_MAX + 1]; + snprintfz(context, RRD_ID_LENGTH_MAX, "netdata.%s.category.memory", c->context_prefix); + + c->st_memory = rrdset_create_localhost( + "netdata" + , id + , NULL + , c->family + , context + , "Dictionary Memory" + , "bytes" + , "netdata" + , "stats" + , c->priority + 4 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); + + c->rd_memory_indexed = rrddim_add(c->st_memory, "index", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + c->rd_memory_values = rrddim_add(c->st_memory, "data", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + c->rd_memory_dict = rrddim_add(c->st_memory, "structures", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + rrdlabels_add(c->st_memory->rrdlabels, "category", stats.name, RRDLABEL_SRC_AUTO); + } + else + rrdset_next(c->st_memory); + + rrddim_set_by_pointer(c->st_memory, c->rd_memory_indexed, (collected_number)stats.memory.indexed); + rrddim_set_by_pointer(c->st_memory, c->rd_memory_values, (collected_number)stats.memory.values); + rrddim_set_by_pointer(c->st_memory, c->rd_memory_dict, (collected_number)stats.memory.dict); + + rrdset_done(c->st_memory); + } + + // ------------------------------------------------------------------------ + + total = 0; + load_dictionary_stats_entry(spin_locks.use); + load_dictionary_stats_entry(spin_locks.search); + load_dictionary_stats_entry(spin_locks.insert); + + if(c->st_spins || total != 0) { + if (unlikely(!c->st_spins)) { + char id[RRD_ID_LENGTH_MAX + 1]; + snprintfz(id, RRD_ID_LENGTH_MAX, "%s.%s.spins", c->context_prefix, stats.name); + + char context[RRD_ID_LENGTH_MAX + 1]; + snprintfz(context, RRD_ID_LENGTH_MAX, "netdata.%s.category.spins", c->context_prefix); + + c->st_spins = rrdset_create_localhost( + "netdata" + , id + , NULL + , c->family + , context + , "Dictionary Spins" + , "count" + , "netdata" + , "stats" + , c->priority + 5 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + c->rd_spins_use = rrddim_add(c->st_spins, "use", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_spins_search = rrddim_add(c->st_spins, "search", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + c->rd_spins_insert = rrddim_add(c->st_spins, "insert", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrdlabels_add(c->st_spins->rrdlabels, "category", stats.name, RRDLABEL_SRC_AUTO); + } + else + rrdset_next(c->st_spins); + + rrddim_set_by_pointer(c->st_spins, c->rd_spins_use, (collected_number)stats.spin_locks.use); + rrddim_set_by_pointer(c->st_spins, c->rd_spins_search, (collected_number)stats.spin_locks.search); + rrddim_set_by_pointer(c->st_spins, c->rd_spins_insert, (collected_number)stats.spin_locks.insert); + + rrdset_done(c->st_spins); + } +} + +static void dictionary_statistics(void) { + for(int i = 0; dictionary_categories[i].stats ;i++) { + update_dictionary_category_charts(&dictionary_categories[i]); + } +} + // --------------------------------------------------------------------------------------------------------------------- // worker utilization @@ -1334,6 +1696,7 @@ static struct worker_utilization all_workers_utilization[] = { { .name = "TIMEX", .family = "workers plugin timex", .priority = 1000000 }, { .name = "IDLEJITTER", .family = "workers plugin idlejitter", .priority = 1000000 }, { .name = "RRDCONTEXT", .family = "workers contexts", .priority = 1000000 }, + { .name = "SERVICE", .family = "workers service", .priority = 1000000 }, // has to be terminated with a NULL { .name = NULL, .family = NULL } @@ -2011,6 +2374,7 @@ void *global_statistics_main(void *ptr) worker_register_job_name(WORKER_JOB_WORKERS, "workers"); worker_register_job_name(WORKER_JOB_DBENGINE, "dbengine"); worker_register_job_name(WORKER_JOB_STRINGS, "strings"); + worker_register_job_name(WORKER_JOB_DICTIONARIES, "dictionaries"); netdata_thread_cleanup_push(global_statistics_cleanup, ptr); @@ -2048,6 +2412,9 @@ void *global_statistics_main(void *ptr) worker_is_busy(WORKER_JOB_STRINGS); update_strings_charts(); + + worker_is_busy(WORKER_JOB_DICTIONARIES); + dictionary_statistics(); } netdata_thread_cleanup_pop(1); diff --git a/daemon/main.c b/daemon/main.c index ada3c14f2a..a51e4a94c8 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -1004,6 +1004,7 @@ int main(int argc, char **argv) { if(test_dbengine()) return 1; #endif if(test_sqlite()) return 1; + if(string_unittest(10000)) return 1; if (dictionary_unittest(10000)) return 1; if (rrdlabels_unittest()) @@ -1028,6 +1029,9 @@ int main(int argc, char **argv) { else if(strcmp(optarg, "dicttest") == 0) { return dictionary_unittest(10000); } + else if(strcmp(optarg, "stringtest") == 0) { + return string_unittest(10000); + } else if(strcmp(optarg, "rrdlabelstest") == 0) { return rrdlabels_unittest(); } diff --git a/daemon/service.c b/daemon/service.c index 61cc1281ae..f8371103b4 100644 --- a/daemon/service.c +++ b/daemon/service.c @@ -5,12 +5,238 @@ /* Run service jobs every X seconds */ #define SERVICE_HEARTBEAT 10 +#define WORKER_JOB_CHILD_CHART_OBSOLETION_CHECK 1 +#define WORKER_JOB_CLEANUP_OBSOLETE_CHARTS 2 +#define WORKER_JOB_ARCHIVE_CHART 3 +#define WORKER_JOB_ARCHIVE_CHART_DIMENSIONS 4 +#define WORKER_JOB_ARCHIVE_DIMENSION 5 +#define WORKER_JOB_CLEANUP_ORPHAN_HOSTS 6 +#define WORKER_JOB_CLEANUP_OBSOLETE_CHARTS_ON_HOSTS 7 +#define WORKER_JOB_FREE_HOST 9 +#define WORKER_JOB_SAVE_HOST_CHARTS 10 +#define WORKER_JOB_DELETE_HOST_CHARTS 11 +#define WORKER_JOB_FREE_CHART 12 +#define WORKER_JOB_SAVE_CHART 13 +#define WORKER_JOB_DELETE_CHART 14 +#define WORKER_JOB_FREE_DIMENSION 15 + +static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) { + RRDSET *st = rd->rrdset; + + if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED | RRDDIM_FLAG_ACLK) || !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) + return; + + worker_is_busy(WORKER_JOB_ARCHIVE_DIMENSION); + + rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED); + rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE); + + const char *cache_filename = rrddim_cache_filename(rd); + if(cache_filename) { + info("Deleting dimension file '%s'.", cache_filename); + if (unlikely(unlink(cache_filename) == -1)) + error("Cannot delete dimension file '%s'", cache_filename); + } + + if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + rrddimvar_delete_all(rd); + + /* only a collector can mark a chart as obsolete, so we must remove the reference */ + + size_t tiers_available = 0, tiers_said_yes = 0; + for(int tier = 0; tier < storage_tiers ;tier++) { + if(rd->tiers[tier]) { + tiers_available++; + + if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle)) + tiers_said_yes++; + + rd->tiers[tier]->db_collection_handle = NULL; + } + } + + if (tiers_available == tiers_said_yes && tiers_said_yes) { + /* This metric has no data and no references */ + delete_dimension_uuid(&rd->metric_uuid); + } + else { + /* Do not delete this dimension */ +#ifdef ENABLE_ACLK + queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec())); +#endif + return; + } + } + + worker_is_busy(WORKER_JOB_FREE_DIMENSION); + rrddim_free(st, rd); +} + +static void svc_rrdset_archive_obsolete_dimensions(RRDSET *st, bool all_dimensions) { + worker_is_busy(WORKER_JOB_ARCHIVE_CHART_DIMENSIONS); + + RRDDIM *rd; + time_t now = now_realtime_sec(); + + dfe_start_reentrant(st->rrddim_root_index, rd) { + if(unlikely( + all_dimensions || + (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && (rd->last_collected_time.tv_sec + rrdset_free_obsolete_time < now)) + )) { + + info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rrddim_name(rd), rrddim_id(rd), rrdset_name(st), rrdset_id(st)); + svc_rrddim_obsolete_to_archive(rd); + + } + } + dfe_done(rd); +} + +static void svc_rrdset_obsolete_to_archive(RRDSET *st) { + worker_is_busy(WORKER_JOB_ARCHIVE_CHART); + + rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED); + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); + + rrdcalc_unlink_all_rrdset_alerts(st); + + svc_rrdset_archive_obsolete_dimensions(st, true); + + rrdsetvar_release_and_delete_all(st); + + // has to be run after all dimensions are archived - or use-after-free will occur + rrdvar_delete_all(st->rrdvars); + + if(st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) { + if(rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS)) { + worker_is_busy(WORKER_JOB_DELETE_CHART); + rrdset_delete_files(st); + } + else { + worker_is_busy(WORKER_JOB_SAVE_CHART); + rrdset_save(st); + } + + worker_is_busy(WORKER_JOB_FREE_CHART); + rrdset_free(st); + } +} + +static void svc_rrdhost_cleanup_obsolete_charts(RRDHOST *host) { + worker_is_busy(WORKER_JOB_CLEANUP_OBSOLETE_CHARTS); + + time_t now = now_realtime_sec(); + RRDSET *st; + rrdset_foreach_reentrant(st, host) { + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) + && st->last_accessed_time + rrdset_free_obsolete_time < now + && st->last_updated.tv_sec + rrdset_free_obsolete_time < now + && st->last_collected_time.tv_sec + rrdset_free_obsolete_time < now + )) { + svc_rrdset_obsolete_to_archive(st); + } + else if(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) { + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS); + svc_rrdset_archive_obsolete_dimensions(st, false); + } +#ifdef ENABLE_ACLK + else + sql_check_chart_liveness(st); +#endif + } + rrdset_foreach_done(st); +} + +static void svc_rrdset_check_obsoletion(RRDHOST *host) { + worker_is_busy(WORKER_JOB_CHILD_CHART_OBSOLETION_CHECK); + + time_t last_entry_t; + RRDSET *st; + rrdset_foreach_read(st, host) { + last_entry_t = rrdset_last_entry_t(st); + + if(last_entry_t && last_entry_t < host->senders_connect_time) + rrdset_is_obsolete(st); + + } + rrdset_foreach_done(st); +} + +static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() { + worker_is_busy(WORKER_JOB_CLEANUP_OBSOLETE_CHARTS_ON_HOSTS); + + rrd_rdlock(); + + RRDHOST *host; + rrdhost_foreach_read(host) { + + if(rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS|RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS)) { + rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS|RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS); + svc_rrdhost_cleanup_obsolete_charts(host); + } + + if(host != localhost + && host->trigger_chart_obsoletion_check + && ( + ( + host->senders_last_chart_command + && host->senders_last_chart_command + host->health_delay_up_to < now_realtime_sec() + ) + || (host->senders_connect_time + 300 < now_realtime_sec()) + ) + ) { + svc_rrdset_check_obsoletion(host); + host->trigger_chart_obsoletion_check = 0; + + } + } + + rrd_unlock(); +} + +static void svc_rrdhost_cleanup_orphan_hosts(RRDHOST *protected_host) { + worker_is_busy(WORKER_JOB_CLEANUP_ORPHAN_HOSTS); + rrd_wrlock(); + + time_t now = now_realtime_sec(); + + RRDHOST *host; + +restart_after_removal: + rrdhost_foreach_write(host) { + if(rrdhost_should_be_removed(host, protected_host, now)) { + info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", rrdhost_hostname(host), host->machine_guid); + + if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) +#ifdef ENABLE_DBENGINE + /* don't delete multi-host DB host files */ + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0])) +#endif + ) { + worker_is_busy(WORKER_JOB_DELETE_HOST_CHARTS); + rrdhost_delete_charts(host); + } + else { + worker_is_busy(WORKER_JOB_SAVE_HOST_CHARTS); + rrdhost_save_charts(host); + } + + worker_is_busy(WORKER_JOB_FREE_HOST); + rrdhost_free(host, 0); + goto restart_after_removal; + } + } + + rrd_unlock(); +} + static void service_main_cleanup(void *ptr) { struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; debug(D_SYSTEM, "Cleaning up..."); + worker_unregister(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } @@ -20,6 +246,22 @@ static void service_main_cleanup(void *ptr) */ void *service_main(void *ptr) { + worker_register("SERVICE"); + worker_register_job_name(WORKER_JOB_CHILD_CHART_OBSOLETION_CHECK, "child chart obsoletion check"); + worker_register_job_name(WORKER_JOB_CLEANUP_OBSOLETE_CHARTS, "cleanup obsolete charts"); + worker_register_job_name(WORKER_JOB_ARCHIVE_CHART, "archive chart"); + worker_register_job_name(WORKER_JOB_ARCHIVE_CHART_DIMENSIONS, "archive chart dimensions"); + worker_register_job_name(WORKER_JOB_ARCHIVE_DIMENSION, "archive dimension"); + worker_register_job_name(WORKER_JOB_CLEANUP_ORPHAN_HOSTS, "cleanup orphan hosts"); + worker_register_job_name(WORKER_JOB_CLEANUP_OBSOLETE_CHARTS_ON_HOSTS, "cleanup obsolete charts on all hosts"); + worker_register_job_name(WORKER_JOB_FREE_HOST, "free host"); + worker_register_job_name(WORKER_JOB_SAVE_HOST_CHARTS, "save host charts"); + worker_register_job_name(WORKER_JOB_DELETE_HOST_CHARTS, "delete host charts"); + worker_register_job_name(WORKER_JOB_FREE_CHART, "free chart"); + worker_register_job_name(WORKER_JOB_SAVE_CHART, "save chart"); + worker_register_job_name(WORKER_JOB_DELETE_CHART, "delete chart"); + worker_register_job_name(WORKER_JOB_FREE_DIMENSION, "free dimension"); + netdata_thread_cleanup_push(service_main_cleanup, ptr); heartbeat_t hb; heartbeat_init(&hb); @@ -28,14 +270,11 @@ void *service_main(void *ptr) debug(D_SYSTEM, "Service thread starts"); while (!netdata_exit) { + worker_is_idle(); heartbeat_next(&hb, step); - rrd_cleanup_obsolete_charts(); - - rrd_wrlock(); - rrdhost_cleanup_orphan_hosts_nolock(localhost); - rrd_unlock(); - + svc_rrd_cleanup_obsolete_charts_from_all_hosts(); + svc_rrdhost_cleanup_orphan_hosts(localhost); } netdata_thread_cleanup_pop(1); diff --git a/daemon/unit_test.c b/daemon/unit_test.c index a2cff95493..fdc3b8d6a0 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -1269,27 +1269,27 @@ static int test_variable_renames(void) { fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rrddim_id(rd2), rrddim_name(rd2)); fprintf(stderr, "Renaming chart to CHARTNAME1\n"); - rrdset_set_name(st, "CHARTNAME1"); + rrdset_reset_name(st, "CHARTNAME1"); fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", rrdset_id(st), rrdset_name(st)); fprintf(stderr, "Renaming chart to CHARTNAME2\n"); - rrdset_set_name(st, "CHARTNAME2"); + rrdset_reset_name(st, "CHARTNAME2"); fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", rrdset_id(st), rrdset_name(st)); fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME1\n"); - rrddim_set_name(st, rd1, "DIM1NAME1"); + rrddim_reset_name(st, rd1, "DIM1NAME1"); fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd1), rrddim_name(rd1)); fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME2\n"); - rrddim_set_name(st, rd1, "DIM1NAME2"); + rrddim_reset_name(st, rd1, "DIM1NAME2"); fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd1), rrddim_name(rd1)); fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME1\n"); - rrddim_set_name(st, rd2, "DIM2NAME1"); + rrddim_reset_name(st, rd2, "DIM2NAME1"); fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2)); fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME2\n"); - rrddim_set_name(st, rd2, "DIM2NAME2"); + rrddim_reset_name(st, rd2, "DIM2NAME2"); fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2)); BUFFER *buf = buffer_create(1); @@ -1447,9 +1447,8 @@ int unit_test(long delay, long shift) long increment = 1000; collected_number i = 0; - unsigned long c, dimensions = 0; + unsigned long c, dimensions = rrdset_number_of_dimensions(st); RRDDIM *rd; - for(rd = st->dimensions; rd ; rd = rd->next) dimensions++; for(c = 0; c < 20 ;c++) { i += increment; @@ -1470,8 +1469,10 @@ int unit_test(long delay, long shift) } // prevent it from deleting the dimensions - for(rd = st->dimensions; rd ; rd = rd->next) + rrddim_foreach_read(rd, st) { rd->last_collected_time.tv_sec = st->last_collected_time.tv_sec; + } + rrddim_foreach_done(rd); rrdset_done(st); } @@ -1486,7 +1487,7 @@ int unit_test(long delay, long shift) for(c = 0 ; c < st->counter ; c++) { fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10); - for(rd = st->dimensions; rd ; rd = rd->next) { + rrddim_foreach_read(rd, st) { sn = rd->db[c]; cn = unpack_storage_number(sn); fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rrddim_id(rd), cn, sn); @@ -1508,6 +1509,7 @@ int unit_test(long delay, long shift) ret = 1; } } + rrddim_foreach_done(rd); } if(ret) @@ -1967,7 +1969,11 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] time_retrieved = r->t[c]; // for each dimension - for (j = 0, d = r->st->dimensions; d && j < r->d ; ++j, d = d->next) { + rrddim_foreach_read(d, r->st) { + if(unlikely((int)d_dfe.counter >= r->d)) break; // d_counter is provided by the dictionary dfe + + j = (int)d_dfe.counter; + NETDATA_DOUBLE *cn = &r->v[ c * r->d ]; value = cn[j]; assert(rd[i][j] == d); @@ -1990,6 +1996,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] time_errors++; } } + rrddim_foreach_done(d); } rrdr_free(owa, r); } @@ -2103,7 +2110,11 @@ int test_dbengine(void) time_t time_retrieved = r->t[c]; // for each dimension - for(j = 0, d = r->st->dimensions; d && j < r->d ; ++j, d = d->next) { + rrddim_foreach_read(d, r->st) { + if(unlikely((int)d_dfe.counter >= r->d)) break; // d_counter is provided by the dictionary dfe + + j = (int)d_dfe.counter; + NETDATA_DOUBLE *cn = &r->v[ c * r->d ]; NETDATA_DOUBLE value = cn[j]; assert(rd[i][j] == d); @@ -2126,6 +2137,7 @@ int test_dbengine(void) time_errors++; } } + rrddim_foreach_done(d); } rrdr_free(owa, r); } diff --git a/database/engine/journalfile.c b/database/engine/journalfile.c index 50625d1e01..dc24bf32a5 100644 --- a/database/engine/journalfile.c +++ b/database/engine/journalfile.c @@ -519,7 +519,7 @@ int load_journal_file(struct rrdengine_instance *ctx, struct rrdengine_journalfi journalfile->file = file; journalfile->pos = file_size; journalfile->data = netdata_mmap(path, file_size, MAP_SHARED, 0); - info("Loading journal file \"%s\" using %s.", path, journalfile->data?"using MMAP":"using uv_fs_read"); + info("Loading journal file \"%s\" using %s.", path, journalfile->data?"MMAP":"uv_fs_read"); max_id = iterate_transactions(ctx, journalfile); diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index 5acab14d3b..27cf4ef059 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -138,7 +138,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_insta uuid_copy(rd->metric_uuid, multihost_legacy_uuid); if (unlikely(need_to_store && !ctx->tier)) - (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); + (void)sql_store_dimension(&rd->metric_uuid, &rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); } struct rrdeng_metric_handle *mh = mallocz(sizeof(struct rrdeng_metric_handle)); diff --git a/database/ram/rrddim_mem.c b/database/ram/rrddim_mem.c index 139f997c93..74674dbdbd 100644 --- a/database/ram/rrddim_mem.c +++ b/database/ram/rrddim_mem.c @@ -42,7 +42,7 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle) { struct mem_collect_handle *ch = (struct mem_collect_handle *)collection_handle; RRDDIM *rd = ch->rd; - memset(rd->db, 0, rd->entries * sizeof(storage_number)); + memset(rd->db, 0, rd->rrdset->entries * sizeof(storage_number)); } int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle) { diff --git a/database/rrd.h b/database/rrd.h index f473e308ac..b536bdeda5 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -16,20 +16,21 @@ typedef struct storage_metric_handle STORAGE_METRIC_HANDLE; typedef struct rrdhost RRDHOST; typedef struct rrddim RRDDIM; typedef struct rrdset RRDSET; -typedef struct rrdvar RRDVAR; -typedef struct rrdsetvar RRDSETVAR; -typedef struct rrddimvar RRDDIMVAR; typedef struct rrdcalc RRDCALC; typedef struct rrdcalctemplate RRDCALCTEMPLATE; typedef struct alarm_entry ALARM_ENTRY; typedef struct context_param CONTEXT_PARAM; +typedef struct rrdfamily_acquired RRDFAMILY_ACQUIRED; +typedef struct rrdvar_acquired RRDVAR_ACQUIRED; +typedef struct rrdsetvar_acquired RRDSETVAR_ACQUIRED; +typedef struct rrdcalc_acquired RRDCALC_ACQUIRED; + typedef void *ml_host_t; typedef void *ml_dimension_t; // forward declarations struct rrddim_tier; -struct rrdset_volatile; struct context_param; #ifdef ENABLE_DBENGINE @@ -75,11 +76,6 @@ struct context_param { uint8_t flags; }; -#define META_CHART_UPDATED 1 -#define META_PLUGIN_UPDATED 2 -#define META_MODULE_UPDATED 4 -#define META_CHART_ACTIVATED 8 - #define UPDATE_EVERY 1 #define UPDATE_EVERY_MAX 3600 @@ -122,7 +118,9 @@ typedef enum rrd_memory_mode { RRD_MEMORY_MODE_MAP = 2, RRD_MEMORY_MODE_SAVE = 3, RRD_MEMORY_MODE_ALLOC = 4, - RRD_MEMORY_MODE_DBENGINE = 5 + RRD_MEMORY_MODE_DBENGINE = 5, + + // this is 8-bit } RRD_MEMORY_MODE; #define RRD_MEMORY_MODE_NONE_NAME "none" @@ -145,7 +143,9 @@ typedef enum rrd_algorithm { RRD_ALGORITHM_ABSOLUTE = 0, RRD_ALGORITHM_INCREMENTAL = 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL = 2, - RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL = 3 + RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL = 3, + + // this is 8-bit } RRD_ALGORITHM; #define RRD_ALGORITHM_ABSOLUTE_NAME "absolute" @@ -159,34 +159,42 @@ extern const char *rrd_algorithm_name(RRD_ALGORITHM algorithm); // ---------------------------------------------------------------------------- // RRD FAMILY -struct rrdfamily { - STRING *family; - DICTIONARY *rrdvar_root_index; - - size_t use_count; -}; -typedef struct rrdfamily RRDFAMILY; +extern const RRDFAMILY_ACQUIRED *rrdfamily_add_and_acquire(RRDHOST *host, const char *id); +extern void rrdfamily_release(RRDHOST *host, const RRDFAMILY_ACQUIRED *rfa); +extern void rrdfamily_index_init(RRDHOST *host); +extern void rrdfamily_index_destroy(RRDHOST *host); +extern DICTIONARY *rrdfamily_rrdvars_dict(const RRDFAMILY_ACQUIRED *rf); // ---------------------------------------------------------------------------- -// flags -// use this for configuration flags, not for state control -// flags are set/unset in a manner that is not thread safe -// and may lead to missing information. +// flags & options +// options are permanent configuration options (no atomics to alter/access them) +typedef enum rrddim_options { + RRDDIM_OPTION_NONE = 0, + RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers + RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers + + // this is 8-bit +} RRDDIM_OPTIONS; + +#define rrddim_option_check(rd, flag) ((rd)->flags & (flag)) +#define rrddim_option_set(rd, flag) (rd)->flags |= (flag) +#define rrddim_option_clear(rd, flag) (rd)->flags &= ~(flag) + +// flags are runtime changing status flags (atomics are required to alter/access them) typedef enum rrddim_flags { RRDDIM_FLAG_NONE = 0, - RRDDIM_FLAG_HIDDEN = (1 << 0), // this dimension will not be offered to callers - RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers RRDDIM_FLAG_OBSOLETE = (1 << 2), // this is marked by the collector/module as obsolete - // No new values have been collected for this dimension since agent start or it was marked RRDDIM_FLAG_OBSOLETE at + // No new values have been collected for this dimension since agent start, or it was marked RRDDIM_FLAG_OBSOLETE at // least rrdset_free_obsolete_time seconds ago. RRDDIM_FLAG_ARCHIVED = (1 << 3), RRDDIM_FLAG_ACLK = (1 << 4), - RRDDIM_FLAG_PENDING_FOREACH_ALARM = (1 << 5), // set when foreach alarm has not been initialized yet + RRDDIM_FLAG_PENDING_FOREACH_ALARMS = (1 << 5), // set when foreach alarm has not been initialized yet RRDDIM_FLAG_META_HIDDEN = (1 << 6), // Status of hidden option in the metadata database - RRDDIM_FLAG_INDEXED_ID = (1 << 7), + + // this is 8 bit } RRDDIM_FLAGS; #define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_SEQ_CST) & (flag)) @@ -231,6 +239,7 @@ extern void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src); void reload_host_labels(void); extern void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels); +extern void rrdset_save_rrdlabels_to_sql(RRDSET *st); extern int rrdlabels_unittest(void); @@ -244,13 +253,15 @@ struct rrddim { uuid_t metric_uuid; // global UUID for this metric (unique_across hosts) // ------------------------------------------------------------------------ - // the dimension definition + // dimension definition STRING *id; // the id of this dimension (for internal identification) STRING *name; // the name of this dimension (as presented to user) - RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values - RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension - RRDDIM_FLAGS flags; // configuration flags for the dimension + + RRD_ALGORITHM algorithm:8; // the algorithm that is applied to add new collected values + RRDDIM_OPTIONS options:8; // permanent configuration options + RRD_MEMORY_MODE rrd_memory_mode:8; // the memory mode for this dimension + /*RRDDIM_FLAGS*/ uint8_t flags; // run time changing status flags bool updated; // 1 when the dimension has been updated since the last processing bool exposed; // 1 when set what have sent this dimension to the central netdata @@ -258,20 +269,38 @@ struct rrddim { collected_number multiplier; // the multiplier of the collected values collected_number divisor; // the divider of the collected values - // ------------------------------------------------------------------------ - // members for temporary data we need for calculations + int update_every; // every how many seconds is this updated + // TODO - remove update_every from rrddim + // it is always the same in rrdset - struct timeval last_collected_time; // when was this dimension last updated - // this is actual date time we updated the last_collected_value - // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET + // ------------------------------------------------------------------------ + // operational state members #ifdef ENABLE_ACLK int aclk_live_status; #endif - ml_dimension_t ml_dimension; + + ml_dimension_t ml_dimension; // machine learning data about this dimension + + // ------------------------------------------------------------------------ + // linking to siblings and parents + + struct rrddim *next; // linking of dimensions within the same data set + struct rrddim *prev; // linking of dimensions within the same data set + + struct rrdset *rrdset; + + RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension + + // ------------------------------------------------------------------------ + // data collection members struct rrddim_tier *tiers[RRD_STORAGE_TIERS]; // our tiers of databases + struct timeval last_collected_time; // when was this dimension last updated + // this is actual date time we updated the last_collected_value + // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET + size_t collections_counter; // the number of times we added values to this rrddim collected_number collected_value_max; // the absolute maximum of the collected value @@ -282,33 +311,12 @@ struct rrddim { collected_number collected_value; // the current value, as collected - resets to 0 after being used collected_number last_collected_value; // the last value that was collected, after being processed - // the *_volume members are used to calculate the accuracy of the rounding done by the - // storage number - they are printed to debug.log when debug is enabled for a set. - NETDATA_DOUBLE collected_volume; // the sum of all collected values so far - NETDATA_DOUBLE stored_volume; // the sum of all stored values so far - - struct rrddim *next; // linking of dimensions within the same data set - struct rrddim *prev; // linking of dimensions within the same data set - - struct rrdset *rrdset; - RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension - // ------------------------------------------------------------------------ - // members for checking the data when loading from disk - - long entries; // how many entries this dimension has in ram - // this is the same to the entries of the data set - // we set it here, to check the data when we load it from disk. - - int update_every; // every how many seconds is this updated + // db mode RAM, SAVE, MAP, ALLOC, NONE specifics + // TODO - they should be managed by storage engine + // (RRDDIM_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) size_t memsize; // the memory allocated for this dimension (without RRDDIM) - - struct rrddimvar *variables; - - // ------------------------------------------------------------------------ - // the values stored in this dimension, using our floating point numbers - void *rd_on_file; // pointer to the header written on disk storage_number *db; // the array of values }; @@ -346,7 +354,6 @@ struct rrddim_query_handle { RRDDIM *rd; time_t start_time; time_t end_time; - TIER_QUERY_FETCH tier_query_fetch_type; STORAGE_QUERY_HANDLE* handle; }; @@ -399,7 +406,7 @@ struct rrddim_collect_ops { // run this to flush / reset the current data collection sequence void (*flush)(STORAGE_COLLECT_HANDLE *collection_handle); - // an finalization function to run after collection is over + // a finalization function to run after collection is over // returns 1 if it's safe to delete the dimension int (*finalize)(STORAGE_COLLECT_HANDLE *collection_handle); }; @@ -432,7 +439,6 @@ struct rrddim_query_ops { struct rrddim_tier { int tier_grouping; RRD_MEMORY_MODE mode; // the memory mode of this tier - RRD_BACKFILL backfill; // backfilling configuration STORAGE_METRIC_HANDLE *db_metric_handle; // the metric handle inside the database STORAGE_COLLECT_HANDLE *db_collection_handle; // the data collection handle STORAGE_POINT virtual_point; @@ -448,11 +454,16 @@ extern void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t n // these loop macros make sure the linked list is accessed with the right lock #define rrddim_foreach_read(rd, st) \ - for((rd) = (st)->dimensions, rrdset_check_rdlock(st); (rd) ; (rd) = (rd)->next) + dfe_start_read((st)->rrddim_root_index, rd) #define rrddim_foreach_write(rd, st) \ - for((rd) = (st)->dimensions, rrdset_check_wrlock(st); (rd) ; (rd) = (rd)->next) + dfe_start_write((st)->rrddim_root_index, rd) +#define rrddim_foreach_reentrant(rd, st) \ + dfe_start_reentrant((st)->rrddim_root_index, rd) + +#define rrddim_foreach_done(rd) \ + dfe_done(rd) // ---------------------------------------------------------------------------- // RRDSET - this is a chart @@ -477,7 +488,7 @@ typedef enum rrdset_flags { RRDSET_FLAG_HIDDEN = (1 << 12), // if set, do not show this chart on the dashboard, but use it for exporting RRDSET_FLAG_SYNC_CLOCK = (1 << 13), // if set, microseconds on next data collection will be ignored (the chart will be synced to now) RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 14), // this is marked by the collector/module when a chart has obsolete dimensions - // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at + // No new values have been collected for this chart since agent start, or it was marked RRDSET_FLAG_OBSOLETE at // least rrdset_free_obsolete_time seconds ago. RRDSET_FLAG_ARCHIVED = (1 << 15), RRDSET_FLAG_ACLK = (1 << 16), @@ -496,14 +507,19 @@ typedef enum rrdset_flags { #define rrdset_is_ar_chart(st) rrdset_flag_check(st, RRDSET_FLAG_ANOMALY_RATE_CHART) struct rrdset { - uuid_t uuid; + uuid_t chart_uuid; // the global UUID for this chart // ------------------------------------------------------------------------ - // the set configuration + // chart configuration - STRING *id; // the ID of the data set - STRING *name; // the name of this dimension (as presented to user) - STRING *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) + struct { + STRING *type; // the type of {type}.{id} + STRING *id; // the id of {type}.{id} + STRING *name; // the name of {type}.{name} + } parts; + + STRING *id; // the unique ID of the rrdset as {type}.{id} + STRING *name; // the unique name of the rrdset as {type}.{name} STRING *family; // grouping sets under the same family STRING *title; // title shown to user STRING *units; // units of measurement @@ -511,44 +527,51 @@ struct rrdset { STRING *plugin_name; // the name of the plugin that generated this STRING *module_name; // the name of the plugin module that generated this - RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart - RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to - - RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset RRDSET_TYPE chart_type; // line, area, stacked - RRDSET_FLAGS flags; // configuration flags - RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances - - int update_every; // every how many seconds is this updated? - - int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored - // netdata will interpolate values for gaps lower than this - - long entries; // total number of entries in the data set - - long current_entry; // the entry that is currently being updated - // it goes around in a round-robin fashion long priority; // the sorting priority of this chart + int update_every; // data collection frequency + + DICTIONARY *rrdlabels; // chart labels + DICTIONARY *rrdsetvar_root_index; // chart variables + DICTIONARY *rrddimvar_root_index; // dimension variables + // we use this dictionary to manage their allocation + + // TODO - dimensions linked list and lock to be removed + netdata_rwlock_t rrdset_rwlock; // protects the dimensions linked list + RRDDIM *dimensions; // chart metrics // ------------------------------------------------------------------------ - // members for temporary data we need for calculations + // operational state members - char *cache_dir; // the directory to store dimensions + RRDSET_FLAGS flags; // flags + RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset + + uuid_t hash_uuid; // hash_id for syncing with cloud + // TODO - obsolete now - cleanup + + DICTIONARY *rrddim_root_index; // dimensions index + + int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored + // netdata will interpolate values for gaps lower than this + // TODO - use the global - all charts have the same value + + // ------------------------------------------------------------------------ + // linking to siblings and parents + + RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to + + RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart + RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to + + // ------------------------------------------------------------------------ + // data collection members size_t counter; // the number of times we added values to this database size_t counter_done; // the number of times rrdset_done() has been called - union { - time_t last_accessed_time; // the last time this RRDSET has been accessed - time_t last_entry_t; // the last_entry_t computed for transient RRDSET - }; - time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream - - uuid_t *chart_uuid; // Store the global GUID for this chart - // this object. - size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine + time_t last_accessed_time; // the last time this RRDSET has been accessed usec_t usec_since_last_update; // the time in microseconds since the last collection of data @@ -558,46 +581,73 @@ struct rrdset { total_number collected_total; // used internally to calculate percentages total_number last_collected_total; // used internally to calculate percentages - RRDFAMILY *rrdfamily; // pointer to RRDFAMILY this chart belongs to - RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to - - struct rrdset *next; // linking of rrdsets - struct rrdset *prev; // linking of rrdsets + size_t rrdlabels_last_saved_version; // ------------------------------------------------------------------------ - // local variables + // data collection - streaming to parents, temp variables - NETDATA_DOUBLE green; // green threshold for this chart - NETDATA_DOUBLE red; // red threshold for this chart - - DICTIONARY *rrdvar_root_index; // RRDVAR index for this chart - RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs) - RRDCALC *alarms; // RRDCALC linked list for this chart + time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream // ------------------------------------------------------------------------ - // members for checking the data when loading from disk + // context queries temp variables + // TODO - eliminate these + time_t last_entry_t; // the last_entry_t computed for transient RRDSET + + // ------------------------------------------------------------------------ + // dbengine specifics + // TODO - they should be managed by storage engine + // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) + + size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine + + // ------------------------------------------------------------------------ + // db mode SAVE, MAP specifics + // TODO - they should be managed by storage engine + // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) + + char *cache_dir; // the directory to store dimensions unsigned long memsize; // how much mem we have allocated for this (without dimensions) void *st_on_file; // compatibility with V019 RRDSET files // ------------------------------------------------------------------------ - // chart labels + // db mode RAM, SAVE, MAP, ALLOC, NONE specifics + // TODO - they should be managed by storage engine + // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) - DICTIONARY *rrdlabels; + long entries; // total number of entries in the data set + + long current_entry; // the entry that is currently being updated + // it goes around in a round-robin fashion // ------------------------------------------------------------------------ - // the dimensions + // exporting to 3rd party time-series members + // TODO - they should be managed by exporting engine + // (RRDSET_EXPORTING_STATE ptr to an undefined structure, and a call to clean this up during destruction) - DICTIONARY *rrddim_root_index; // the root of the dimensions index + RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances - netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list - RRDDIM *dimensions; // the actual data for every dimension + // ------------------------------------------------------------------------ + // health monitoring members + // TODO - they should be managed by health + // (RRDSET_HEALTH_STATE ptr to an undefined structure, and a call to clean this up during destruction) + + NETDATA_DOUBLE green; // green threshold for this chart + NETDATA_DOUBLE red; // red threshold for this chart + + DICTIONARY *rrdvars; // RRDVAR index for this chart + const RRDFAMILY_ACQUIRED *rrdfamily; // pointer to RRDFAMILY dictionary item, this chart belongs to + + struct { + netdata_rwlock_t rwlock; // protection for RRDCALC *base + RRDCALC *base; // double linked list of RRDCALC related to this RRDSET + } alerts; }; #define rrdset_plugin_name(st) string2str((st)->plugin_name) #define rrdset_module_name(st) string2str((st)->module_name) #define rrdset_units(st) string2str((st)->units) -#define rrdset_type(st) string2str((st)->type) +#define rrdset_parts_type(st) string2str((st)->parts.type) #define rrdset_family(st) string2str((st)->family) #define rrdset_title(st) string2str((st)->title) #define rrdset_context(st) string2str((st)->context) @@ -614,11 +664,19 @@ extern STRING *rrd_string_strdupz(const char *s); // these loop macros make sure the linked list is accessed with the right lock #define rrdset_foreach_read(st, host) \ - for((st) = (host)->rrdset_root, rrdhost_check_rdlock(host); st ; (st) = (st)->next) + dfe_start_read((host)->rrdset_root_index, st) #define rrdset_foreach_write(st, host) \ - for((st) = (host)->rrdset_root, rrdhost_check_wrlock(host); st ; (st) = (st)->next) + dfe_start_write((host)->rrdset_root_index, st) +#define rrdset_foreach_reentrant(st, host) \ + dfe_start_reentrant((host)->rrdset_root_index, st) + +#define rrdset_foreach_done(st) \ + dfe_done(st) + +#define rrdset_number_of_dimensions(st) \ + dictionary_entries((st)->rrddim_root_index) extern void rrdset_memory_file_save(RRDSET *st); extern void rrdset_memory_file_free(RRDSET *st); @@ -633,19 +691,23 @@ extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY // and may lead to missing information. typedef enum rrdhost_flags { - RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data) - RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts - RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan - RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases - RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases - RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet - RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms - RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8), - RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9), - RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates - RRDHOST_FLAG_INDEXED_MACHINE_GUID = (1 << 11), // when set, we have indexed its machine guid - RRDHOST_FLAG_INDEXED_HOSTNAME = (1 << 12), // when set, we have indexed its hostname - RRDHOST_FLAG_STREAM_COLLECTED_METRICS = (1 << 13), // when set, rrdset_done() should push metrics to parent + RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data) + RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts + RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan + RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases + RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases + RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet + RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms + RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8), + RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9), + RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates + RRDHOST_FLAG_INDEXED_MACHINE_GUID = (1 << 11), // when set, we have indexed its machine guid + RRDHOST_FLAG_INDEXED_HOSTNAME = (1 << 12), // when set, we have indexed its hostname + RRDHOST_FLAG_STREAM_COLLECTED_METRICS = (1 << 13), // when set, rrdset_done() should push metrics to parent + RRDHOST_FLAG_INITIALIZED_HEALTH = (1 << 14), // the host has initialized health structures + RRDHOST_FLAG_INITIALIZED_RRDPUSH = (1 << 15), // the host has initialized rrdpush structures + RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS = (1 << 16), // the host has pending chart obsoletions + RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS = (1 << 17), // the host has pending dimension obsoletions } RRDHOST_FLAGS; #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag)) @@ -868,27 +930,18 @@ struct rrdhost { // all RRDCALCs are primarily allocated and linked here // RRDCALCs may be linked to charts at any point // (charts may or may not exist when these are loaded) - RRDCALC *host_alarms; - RRDCALC *alarms_with_foreach; + DICTIONARY *rrdcalc_root_index; + + // templates of alarms + // these are used to create alarms when charts + // are created or renamed, that match them + DICTIONARY *rrdcalctemplate_root_index; ALARM_LOG health_log; // alarms historical events (event log) uint32_t health_last_processed_id; // the last processed health id from the log uint32_t health_max_unique_id; // the max alarm log unique id given for the host uint32_t health_max_alarm_id; // the max alarm id given for the host - // templates of alarms - // these are used to create alarms when charts - // are created or renamed, that match them - RRDCALCTEMPLATE *alarms_templates; - - // ------------------------------------------------------------------------ - // the charts of the host - - RRDSET *rrdset_root; // the host charts - - unsigned int obsolete_charts_count; - - // ------------------------------------------------------------------------ // locks @@ -909,7 +962,8 @@ struct rrdhost { DICTIONARY *rrdset_root_index_name; // the host's charts index (by name) DICTIONARY *rrdfamily_root_index; // the host's chart families index - DICTIONARY *rrdvar_root_index; // the host's chart variables index + DICTIONARY *rrdvars; // the host's chart variables index + // this includes custom host variables STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers @@ -973,6 +1027,11 @@ extern netdata_rwlock_t rrd_rwlock; // ---------------------------------------------------------------------------- extern bool is_storage_engine_shared(STORAGE_INSTANCE *engine); +extern void rrdset_index_init(RRDHOST *host); +extern void rrdset_index_destroy(RRDHOST *host); + +extern void rrddim_index_init(RRDSET *st); +extern void rrddim_index_destroy(RRDSET *st); // ---------------------------------------------------------------------------- @@ -1058,7 +1117,7 @@ extern void __rrd_check_wrlock(const char *file, const char *function, const uns // ---------------------------------------------------------------------------- // RRDSET functions -extern int rrdset_set_name(RRDSET *st, const char *name); +extern int rrdset_reset_name(RRDSET *st, const char *name); extern RRDSET *rrdset_create_custom(RRDHOST *host , const char *type @@ -1086,12 +1145,10 @@ extern void rrdhost_free_all(void); extern void rrdhost_save_all(void); extern void rrdhost_cleanup_all(void); -extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host); extern void rrdhost_system_info_free(struct rrdhost_system_info *system_info); extern void rrdhost_free(RRDHOST *host, bool force); extern void rrdhost_save_charts(RRDHOST *host); extern void rrdhost_delete_charts(RRDHOST *host); -extern void rrd_cleanup_obsolete_charts(); extern int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now); @@ -1140,95 +1197,19 @@ extern void rrdset_is_obsolete(RRDSET *st); extern void rrdset_isnot_obsolete(RRDSET *st); // checks if the RRDSET should be offered to viewers -#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE) +#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE) #define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions) #define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions) -// get the timestamp of the last entry in the round robin database -static inline time_t rrddim_last_entry_t(RRDDIM *rd) { - time_t latest = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle); - - for(int tier = 1; tier < storage_tiers ;tier++) { - if(unlikely(!rd->tiers[tier])) continue; - - time_t t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle); - if(t > latest) - latest = t; - } - - return latest; -} - -static inline time_t rrddim_first_entry_t(RRDDIM *rd) { - time_t oldest = 0; - - for(int tier = 0; tier < storage_tiers ;tier++) { - if(unlikely(!rd->tiers[tier])) continue; - - time_t t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle); - if(t != 0 && (oldest == 0 || t < oldest)) - oldest = t; - } - - return oldest; -} - -// get the timestamp of the last entry in the round robin database -static inline time_t rrdset_last_entry_t_nolock(RRDSET *st) { - RRDDIM *rd; - time_t last_entry_t = 0; - - rrddim_foreach_read(rd, st) { - time_t t = rrddim_last_entry_t(rd); - if(t > last_entry_t) last_entry_t = t; - } - - return last_entry_t; -} - -static inline time_t rrdset_last_entry_t(RRDSET *st) { - time_t last_entry_t; - - netdata_rwlock_rdlock(&st->rrdset_rwlock); - last_entry_t = rrdset_last_entry_t_nolock(st); - netdata_rwlock_unlock(&st->rrdset_rwlock); - - return last_entry_t; -} - -// get the timestamp of first entry in the round robin database -static inline time_t rrdset_first_entry_t_nolock(RRDSET *st) { - RRDDIM *rd; - time_t first_entry_t = LONG_MAX; - - rrddim_foreach_read(rd, st) { - time_t t = rrddim_first_entry_t(rd); - if(t < first_entry_t) - first_entry_t = t; - } - - if (unlikely(LONG_MAX == first_entry_t)) return 0; - return first_entry_t; -} - -static inline time_t rrdset_first_entry_t(RRDSET *st) -{ - time_t first_entry_t; - - netdata_rwlock_rdlock(&st->rrdset_rwlock); - first_entry_t = rrdset_first_entry_t_nolock(st); - netdata_rwlock_unlock(&st->rrdset_rwlock); - - return first_entry_t; -} - -time_t rrdhost_last_entry_t(RRDHOST *h); +extern time_t rrddim_first_entry_t(RRDDIM *rd); +extern time_t rrddim_last_entry_t(RRDDIM *rd); +extern time_t rrdset_last_entry_t(RRDSET *st); +extern time_t rrdset_first_entry_t(RRDSET *st); +extern time_t rrdhost_last_entry_t(RRDHOST *h); // ---------------------------------------------------------------------------- // RRD DIMENSION functions -extern void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host); - extern RRDDIM *rrddim_add_custom(RRDSET *st , const char *id , const char *name @@ -1241,7 +1222,7 @@ extern RRDDIM *rrddim_add_custom(RRDSET *st #define rrddim_add(st, id, name, multiplier, divisor, algorithm) \ rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode) -extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name); +extern int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name); extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm); extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier); extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor); @@ -1270,19 +1251,17 @@ extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length); // ---------------------------------------------------------------------------- // RRD internal functions +extern void rrdset_delete_files(RRDSET *st); +extern void rrdset_save(RRDSET *st); +extern void rrdset_free(RRDSET *st); + #ifdef NETDATA_RRD_INTERNALS extern char *rrdset_cache_dir(RRDHOST *host, const char *id); extern void rrddim_free(RRDSET *st, RRDDIM *rd); -extern RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id); -extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc); - -extern void rrdset_free(RRDSET *st); extern void rrdset_reset(RRDSET *st); -extern void rrdset_save(RRDSET *st); -extern void rrdset_delete_files(RRDSET *st); extern void rrdset_delete_obsolete_dimensions(RRDSET *st); extern RRDHOST *rrdhost_create( diff --git a/database/rrdcalc.c b/database/rrdcalc.c index 83dd527325..bd92c7a700 100644 --- a/database/rrdcalc.c +++ b/database/rrdcalc.c @@ -1,10 +1,9 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#define NETDATA_HEALTH_INTERNALS #include "rrd.h" // ---------------------------------------------------------------------------- -// RRDCALC management +// RRDCALC helpers inline const char *rrdcalc_status2string(RRDCALC_STATUS status) { switch(status) { @@ -35,7 +34,38 @@ inline const char *rrdcalc_status2string(RRDCALC_STATUS status) { } } -static STRING *rrdcalc_replace_variables(const char *line, RRDCALC *rc) { +uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id) { + netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); + + // re-use old IDs, by looking them up in the alarm log + ALARM_ENTRY *ae = NULL; + for(ae = host->health_log.alarms; ae ;ae = ae->next) { + if(unlikely(name == ae->name && chart == ae->chart)) { + if(next_event_id) *next_event_id = ae->alarm_event_id + 1; + break; + } + } + + uint32_t alarm_id; + + if(ae) + alarm_id = ae->alarm_id; + + else { + if (unlikely(!host->health_log.next_alarm_id)) + host->health_log.next_alarm_id = (uint32_t)now_realtime_sec(); + + alarm_id = host->health_log.next_alarm_id++; + } + + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); + return alarm_id; +} + +// ---------------------------------------------------------------------------- +// RRDCALC replacing info text variables with RRDSET labels + +static STRING *rrdcalc_replace_variables_with_rrdset_labels(const char *line, RRDCALC *rc) { if (!line || !*line) return NULL; @@ -85,19 +115,62 @@ static STRING *rrdcalc_replace_variables(const char *line, RRDCALC *rc) { return ret; } -void rrdcalc_update_rrdlabels(RRDSET *st) { - RRDCALC *rc; - foreach_rrdcalc_in_rrdset(st, rc) { - if (rc->original_info) { - if (rc->info) - string_freez(rc->info); +void rrdcalc_update_info_using_rrdset_labels(RRDCALC *rc) { + if(!rc->rrdset || !rc->original_info || !rc->rrdset->rrdlabels) return; - rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc); - } + size_t labels_version = dictionary_version(rc->rrdset->rrdlabels); + if(rc->labels_version != labels_version) { + + STRING *old = rc->info; + rc->info = rrdcalc_replace_variables_with_rrdset_labels(rrdcalc_original_info(rc), rc); + string_freez(old); + + rc->labels_version = labels_version; } } -static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { +// ---------------------------------------------------------------------------- +// RRDCALC index management for RRDSET + +// the dictionary requires a unique key for every item +// we use {chart id}.{alert name} for both the RRDHOST and RRDSET alert indexes. + +#define RRDCALC_MAX_KEY_SIZE 1024 +static size_t rrdcalc_key(char *dst, size_t dst_len, const char *chart, const char *alert) { + return snprintfz(dst, dst_len, "%s/%s", chart, alert); +} + +const RRDCALC_ACQUIRED *rrdcalc_from_rrdset_get(RRDSET *st, const char *alert_name) { + char key[RRDCALC_MAX_KEY_SIZE + 1]; + size_t key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_id(st), alert_name); + + const RRDCALC_ACQUIRED *rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)(key_len + 1)); + + if(!rca) { + key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_name(st), alert_name); + rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)(key_len + 1)); + } + + return rca; +} + +void rrdcalc_from_rrdset_release(RRDSET *st, const RRDCALC_ACQUIRED *rca) { + if(!rca) return; + + dictionary_acquired_item_release(st->rrdhost->rrdcalc_root_index, (const DICTIONARY_ITEM *)rca); +} + +RRDCALC *rrdcalc_acquired_to_rrdcalc(const RRDCALC_ACQUIRED *rca) { + if(rca) + return dictionary_acquired_item_value((const DICTIONARY_ITEM *)rca); + + return NULL; +} + +// ---------------------------------------------------------------------------- +// RRDCALC managing the linking with RRDSET + +static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) { RRDHOST *host = st->rrdhost; debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host)); @@ -105,7 +178,9 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { rc->last_status_change = now_realtime_sec(); rc->rrdset = st; - DOUBLE_LINKED_LIST_PREPEND_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next); + netdata_rwlock_wrlock(&st->alerts.rwlock); + DOUBLE_LINKED_LIST_APPEND_UNSAFE(st->alerts.base, rc, prev, next); + netdata_rwlock_unlock(&st->alerts.rwlock); if(rc->update_every < rc->rrdset->update_every) { error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every); @@ -114,40 +189,61 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { if(!isnan(rc->green) && isnan(st->green)) { debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " NETDATA_DOUBLE_FORMAT_AUTO - " to " NETDATA_DOUBLE_FORMAT_AUTO ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->green, rc->green); + " to " NETDATA_DOUBLE_FORMAT_AUTO ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->green, rc->green); st->green = rc->green; } if(!isnan(rc->red) && isnan(st->red)) { debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " NETDATA_DOUBLE_FORMAT_AUTO " to " NETDATA_DOUBLE_FORMAT_AUTO - ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->red, rc->red); + ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->red, rc->red); st->red = rc->red; } - rc->local = rrdvar_create_and_index("local", st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value); - rc->family = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value); + char buf[RRDVAR_MAX_LENGTH + 1]; + snprintfz(buf, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), rrdcalc_name(rc)); + STRING *rrdset_name_rrdcalc_name = string_strdupz(buf); + snprintfz(buf, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), rrdcalc_name(rc)); + STRING *rrdset_id_rrdcalc_name = string_strdupz(buf); - char fullname[RRDVAR_MAX_LENGTH + 1]; - snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), rrdcalc_name(rc)); - STRING *fullname_string = string_strdupz(fullname); - rc->hostid = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value); + rc->rrdvar_local = rrdvar_add_and_acquire( + "local", + st->rrdvars, + rc->name, + RRDVAR_TYPE_CALCULATED, + RRDVAR_FLAG_RRDCALC_LOCAL_VAR, + &rc->value); - snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), rrdcalc_name(rc)); - rc->hostname = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value); + rc->rrdvar_family = rrdvar_add_and_acquire( + "family", + rrdfamily_rrdvars_dict(st->rrdfamily), + rc->name, + RRDVAR_TYPE_CALCULATED, + RRDVAR_FLAG_RRDCALC_FAMILY_VAR, + &rc->value); - string_freez(fullname_string); + rc->rrdvar_host_chart_name = rrdvar_add_and_acquire( + "host", + host->rrdvars, + rrdset_name_rrdcalc_name, + RRDVAR_TYPE_CALCULATED, + RRDVAR_FLAG_RRDCALC_HOST_CHARTNAME_VAR, + &rc->value); - if(rc->hostid && !rc->hostname) - rc->hostid->options |= RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR; + rc->rrdvar_host_chart_id = rrdvar_add_and_acquire( + "host", + host->rrdvars, + rrdset_id_rrdcalc_name, + RRDVAR_TYPE_CALCULATED, + RRDVAR_FLAG_RRDCALC_HOST_CHARTID_VAR | ((rc->rrdvar_host_chart_name) ? 0 : RRDVAR_FLAG_RRDCALC_HOST_CHARTNAME_VAR), + &rc->value); - if(!rc->units) rc->units = string_dup(st->units); + string_freez(rrdset_id_rrdcalc_name); + string_freez(rrdset_name_rrdcalc_name); - if (rc->original_info) { - if (rc->info) - string_freez(rc->info); + if(!rc->units) + rc->units = string_dup(st->units); - rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc); - } + rrdcalc_update_info_using_rrdset_labels(rc); time_t now = now_realtime_sec(); @@ -177,43 +273,10 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { 0, rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0); - health_alarm_log(host, ae); + health_alarm_log_add_entry(host, ae); } -static inline int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) { - if ( (rc->chart != st->id) - && (rc->chart != st->name)) - return 0; - - if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, rrdset_module_name(st))) - return 0; - - if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, rrdset_plugin_name(st))) - return 0; - - if (st->rrdhost->rrdlabels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->rrdlabels, rc->host_labels_pattern, '=')) - return 0; - - return 1; -} - -// this has to be called while the RRDHOST is locked -inline void rrdsetcalc_link_matching(RRDSET *st) { - RRDHOST *host = st->rrdhost; - // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id); - - RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) { - if(unlikely(rc->rrdset)) - continue; - - if(unlikely(rrdcalc_is_matching_rrdset(rc, st))) - rrdsetcalc_link(st, rc); - } -} - -// this has to be called while the RRDHOST is locked -inline void rrdsetcalc_unlink(RRDCALC *rc) { +static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) { RRDSET *st = rc->rrdset; if(!st) { @@ -252,139 +315,176 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { 0, 0); - health_alarm_log(host, ae); + health_alarm_log_add_entry(host, ae); debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host)); // unlink it - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next); - rrdvar_free(host, st->rrdvar_root_index, rc->local); - rc->local = NULL; + if(!having_ll_wrlock) + netdata_rwlock_wrlock(&st->alerts.rwlock); - rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rc->family); - rc->family = NULL; + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->alerts.base, rc, prev, next); - rrdvar_free(host, host->rrdvar_root_index, rc->hostid); - rc->hostid = NULL; - - rrdvar_free(host, host->rrdvar_root_index, rc->hostname); - rc->hostname = NULL; + if(!having_ll_wrlock) + netdata_rwlock_unlock(&st->alerts.rwlock); rc->rrdset = NULL; + rrdvar_release_and_del(st->rrdvars, rc->rrdvar_local); + rc->rrdvar_local = NULL; + + rrdvar_release_and_del(rrdfamily_rrdvars_dict(st->rrdfamily), rc->rrdvar_family); + rc->rrdvar_family = NULL; + + rrdvar_release_and_del(host->rrdvars, rc->rrdvar_host_chart_id); + rc->rrdvar_host_chart_id = NULL; + + rrdvar_release_and_del(host->rrdvars, rc->rrdvar_host_chart_name); + rc->rrdvar_host_chart_name = NULL; + // RRDCALC will remain in RRDHOST // so that if the matching chart is found in the future // it will be applied automatically } -RRDCALC *rrdcalc_find(RRDSET *st, const char *name) { - RRDCALC *rc = NULL; +static inline bool rrdcalc_check_if_it_matches_rrdset(RRDCALC *rc, RRDSET *st) { + if ( (rc->chart != st->id) + && (rc->chart != st->name)) + return false; - STRING *name_string = string_strdupz(name); + if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, rrdset_module_name(st))) + return false; - foreach_rrdcalc_in_rrdset(st, rc) { - if(unlikely(rc->name == name_string)) - break; - } + if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, rrdset_plugin_name(st))) + return false; - string_freez(name_string); + if (st->rrdhost->rrdlabels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->rrdlabels, rc->host_labels_pattern, '=')) + return false; - return rc; + return true; } -inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name) { +void rrdcalc_link_matching_alerts_to_rrdset(RRDSET *st) { + RRDHOST *host = st->rrdhost; + // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id); + RRDCALC *rc; + foreach_rrdcalc_in_rrdhost_read(host, rc) { + if(rc->rrdset) + continue; - if(unlikely(!chart)) { - error("attempt to find RRDCALC '%s' without giving a chart name", name); - return 1; + if(unlikely(rrdcalc_check_if_it_matches_rrdset(rc, st))) + rrdcalc_link_to_rrdset(st, rc); + } + foreach_rrdcalc_in_rrdhost_done(rc); +} + +static inline int rrdcalc_check_and_link_rrdset_callback(RRDSET *st, void *rrdcalc) { + RRDCALC *rc = rrdcalc; + + if(unlikely(rrdcalc_check_if_it_matches_rrdset(rc, st))) { + rrdcalc_link_to_rrdset(st, rc); + return -1; } - STRING *name_string = string_strdupz(name); - STRING *chart_string = string_strdupz(chart); - int ret = 0; + return 0; +} - // make sure it does not already exist - foreach_rrdcalc_in_rrdhost(host, rc) { - if (unlikely(rc->chart == chart_string && rc->name == name_string)) { - debug(D_HEALTH, "Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host)); - info("Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host)); - ret = 1; - break; +// ---------------------------------------------------------------------------- +// RRDCALC rrdhost index management - constructor + +struct rrdcalc_constructor { + RRDHOST *rrdhost; // the host we operate upon + RRDCALC *from_config; // points to the original RRDCALC, as loaded from the config + RRDCALCTEMPLATE *from_rrdcalctemplate; // the template this alert is generated from + RRDSET *rrdset; // when this comes from rrdcalctemplate, we have a matching rrdset + const char *overwrite_alert_name; // when we have a dimension foreach, the alert is renamed + const char *overwrite_dimensions; // when we have a dimension foreach, the dimensions filter is renamed + + enum { + RRDCALC_REACT_NONE, + RRDCALC_REACT_NEW, + } react_action; +}; + +static void rrdcalc_rrdhost_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *constructor_data) { + RRDCALC *rc = rrdcalc; + struct rrdcalc_constructor *ctr = constructor_data; + RRDHOST *host = ctr->rrdhost; + + rc->key = string_strdupz(dictionary_acquired_item_name(item)); + + if(ctr->from_rrdcalctemplate) { + rc->run_flags |= RRDCALC_FLAG_FROM_TEMPLATE; + + RRDCALCTEMPLATE *rt = ctr->from_rrdcalctemplate; + RRDSET *st = ctr->rrdset; + + rc->next_event_id = 1; + rc->name = (ctr->overwrite_alert_name) ? string_strdupz(ctr->overwrite_alert_name) : string_dup(rt->name); + rc->chart = string_dup(st->id); + uuid_copy(rc->config_hash_id, rt->config_hash_id); + + rc->dimensions = (ctr->overwrite_dimensions) ? string_strdupz(ctr->overwrite_dimensions) : string_dup(rt->dimensions); + rc->foreach_dimension = NULL; + rc->foreach_dimension_pattern = NULL; + + rc->green = rt->green; + rc->red = rt->red; + rc->value = NAN; + rc->old_value = NAN; + + rc->delay_up_duration = rt->delay_up_duration; + rc->delay_down_duration = rt->delay_down_duration; + rc->delay_max_duration = rt->delay_max_duration; + rc->delay_multiplier = rt->delay_multiplier; + + rc->last_repeat = 0; + rc->times_repeat = 0; + rc->warn_repeat_every = rt->warn_repeat_every; + rc->crit_repeat_every = rt->crit_repeat_every; + + rc->group = rt->group; + rc->after = rt->after; + rc->before = rt->before; + rc->update_every = rt->update_every; + rc->options = rt->options; + + rc->exec = string_dup(rt->exec); + rc->recipient = string_dup(rt->recipient); + rc->source = string_dup(rt->source); + rc->units = string_dup(rt->units); + rc->info = string_dup(rt->info); + rc->original_info = string_dup(rt->info); + + rc->classification = string_dup(rt->classification); + rc->component = string_dup(rt->component); + rc->type = string_dup(rt->type); + + if(rt->calculation) { + rc->calculation = expression_parse(rt->calculation->source, NULL, NULL); + if(!rc->calculation) + error("Health alarm '%s.%s': failed to parse calculation expression '%s'", rrdset_id(st), rrdcalctemplate_name(rt), rt->calculation->source); + } + if(rt->warning) { + rc->warning = expression_parse(rt->warning->source, NULL, NULL); + if(!rc->warning) + error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", rrdset_id(st), rrdcalctemplate_name(rt), rt->warning->source); + } + if(rt->critical) { + rc->critical = expression_parse(rt->critical->source, NULL, NULL); + if(!rc->critical) + error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", rrdset_id(st), rrdcalctemplate_name(rt), rt->critical->source); } } - - string_freez(name_string); - string_freez(chart_string); - - return ret; -} - -inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id) { - // re-use old IDs, by looking them up in the alarm log - ALARM_ENTRY *ae = NULL; - for(ae = host->health_log.alarms; ae ;ae = ae->next) { - if(unlikely(name == ae->name && chart == ae->chart)) { - if(next_event_id) *next_event_id = ae->alarm_event_id + 1; - break; - } + else if(ctr->from_config) { + // dictionary has already copied all the members values and pointers + // no need for additional work in this case + ; } - if(ae) - return ae->alarm_id; - - if (unlikely(!host->health_log.next_alarm_id)) - host->health_log.next_alarm_id = (uint32_t)now_realtime_sec(); - - return host->health_log.next_alarm_id++; -} - -/** - * Alarm name with dimension - * - * Change the name of the current alarm appending a new diagram. - * - * @param name the alarm name - * @param namelen is the length of the previous vector. - * @param dim the dimension of the chart. - * @param dimlen is the length of the previous vector. - * - * @return It returns the new name on success and the old otherwise - */ -char *alarm_name_with_dim(const char *name, size_t namelen, const char *dim, size_t dimlen) { - char *newname,*move; - - newname = mallocz(namelen + dimlen + 2); - move = newname; - memcpy(move, name, namelen); - move += namelen; - - *move++ = '_'; - memcpy(move, dim, dimlen); - move += dimlen; - *move = '\0'; - - return newname; -} - -/** - * Remove pipe comma - * - * Remove the pipes and commas converting to space. - * - * @param str the string to change. - */ -void dimension_remove_pipe_comma(char *str) { - while(*str) { - if(*str == '|' || *str == ',') *str = ' '; - - str++; - } -} - -inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) { - rrdhost_check_rdlock(host); + rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id); if(rc->calculation) { rc->calculation->status = &rc->status; @@ -410,218 +510,66 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) { rc->critical->rrdcalc = rc; } - if(!rc->foreachdim) { - // link it to the host alarms list - DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->host_alarms, rc, prev, next); - - // link it to its chart - RRDSET *st; - rrdset_foreach_read(st, host) { - if(rrdcalc_is_matching_rrdset(rc, st)) { - rrdsetcalc_link(st, rc); - break; - } - } - } - else { - DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->alarms_with_foreach, rc, prev, next); - // We are not linking this alarm directly to the host here - // It will eventually be done if it matches the dimensions - } -} - -inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) { - debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rrdcalctemplate_name(rt)); - - if(rrdcalc_exists(host, chart, rrdcalctemplate_name(rt))) - return NULL; - - RRDCALC *rc = callocz(1, sizeof(RRDCALC)); - rc->next_event_id = 1; - rc->name = string_dup(rt->name); - rc->chart = string_strdupz(chart); - uuid_copy(rc->config_hash_id, rt->config_hash_id); - - rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id); - - rc->dimensions = string_dup(rt->dimensions); - rc->foreachdim = string_dup(rt->foreachdim); - if(rt->foreachdim) - rc->spdim = health_pattern_from_foreach(rrdcalc_foreachdim(rc)); - - rc->foreachcounter = rt->foreachcounter; - - rc->green = rt->green; - rc->red = rt->red; - rc->value = NAN; - rc->old_value = NAN; - - rc->delay_up_duration = rt->delay_up_duration; - rc->delay_down_duration = rt->delay_down_duration; - rc->delay_max_duration = rt->delay_max_duration; - rc->delay_multiplier = rt->delay_multiplier; - - rc->last_repeat = 0; - rc->times_repeat = 0; - rc->warn_repeat_every = rt->warn_repeat_every; - rc->crit_repeat_every = rt->crit_repeat_every; - - rc->group = rt->group; - rc->after = rt->after; - rc->before = rt->before; - rc->update_every = rt->update_every; - rc->options = rt->options; - - rc->exec = string_dup(rt->exec); - rc->recipient = string_dup(rt->recipient); - rc->source = string_dup(rt->source); - rc->units = string_dup(rt->units); - rc->info = string_dup(rt->info); - rc->original_info = string_dup(rt->info); - - rc->classification = string_dup(rt->classification); - rc->component = string_dup(rt->component); - rc->type = string_dup(rt->type); - - if(rt->calculation) { - rc->calculation = expression_parse(rt->calculation->source, NULL, NULL); - if(!rc->calculation) - error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, rrdcalctemplate_name(rt), rt->calculation->source); - } - if(rt->warning) { - rc->warning = expression_parse(rt->warning->source, NULL, NULL); - if(!rc->warning) - error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, rrdcalctemplate_name(rt), rt->warning->source); - } - if(rt->critical) { - rc->critical = expression_parse(rt->critical->source, NULL, NULL); - if(!rc->critical) - error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rrdcalctemplate_name(rt), rt->critical->source); - } - - debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO - ", red " NETDATA_DOUBLE_FORMAT_AUTO - ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", - rrdcalc_chart_name(rc), - rrdcalc_name(rc), - (rc->exec)?rrdcalc_exec(rc):"DEFAULT", - (rc->recipient)?rrdcalc_recipient(rc):"DEFAULT", - rc->green, - rc->red, - (int)rc->group, - rc->after, - rc->before, - rc->options, - (rc->dimensions)?rrdcalc_dimensions(rc):"NONE", - (rc->foreachdim)?rrdcalc_foreachdim(rc):"NONE", - rc->update_every, - (rc->calculation)?rc->calculation->parsed_as:"NONE", - (rc->warning)?rc->warning->parsed_as:"NONE", - (rc->critical)?rc->critical->parsed_as:"NONE", - rrdcalc_source(rc), - rc->delay_up_duration, - rc->delay_down_duration, - rc->delay_max_duration, - rc->delay_multiplier, - rc->warn_repeat_every, - rc->crit_repeat_every + debug(D_HEALTH, "Health added alarm '%s.%s': exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO + ", red " NETDATA_DOUBLE_FORMAT_AUTO + ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", + rrdcalc_chart_name(rc), + rrdcalc_name(rc), + (rc->exec)?rrdcalc_exec(rc):"DEFAULT", + (rc->recipient)?rrdcalc_recipient(rc):"DEFAULT", + rc->green, + rc->red, + (int)rc->group, + rc->after, + rc->before, + rc->options, + (rc->dimensions)?rrdcalc_dimensions(rc):"NONE", + (rc->foreach_dimension)?rrdcalc_foreachdim(rc):"NONE", + rc->update_every, + (rc->calculation)?rc->calculation->parsed_as:"NONE", + (rc->warning)?rc->warning->parsed_as:"NONE", + (rc->critical)?rc->critical->parsed_as:"NONE", + rrdcalc_source(rc), + rc->delay_up_duration, + rc->delay_down_duration, + rc->delay_max_duration, + rc->delay_multiplier, + rc->warn_repeat_every, + rc->crit_repeat_every ); - rrdcalc_add_to_host(host, rc); - return rc; + ctr->react_action = RRDCALC_REACT_NEW; } -/** - * Create from RRDCALC - * - * Create a new alarm using another alarm as template. - * - * @param rc is the alarm that will be used as source - * @param host is the host structure. - * @param name is the newest chart name. - * @param dimension is the current dimension - * @param foreachdim the whole list of dimension - * - * @return it returns the new alarm changed. - */ -inline RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const char *name, const char *dimension) { - RRDCALC *newrc = callocz(1, sizeof(RRDCALC)); +static void rrdcalc_rrdhost_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *constructor_data) { + RRDCALC *rc = rrdcalc; + struct rrdcalc_constructor *ctr = constructor_data; + RRDHOST *host = ctr->rrdhost; - newrc->next_event_id = 1; - newrc->name = string_strdupz(name); - newrc->chart = string_dup(rc->chart); - newrc->id = rrdcalc_get_unique_id(host, newrc->chart, newrc->name, &rc->next_event_id); - uuid_copy(newrc->config_hash_id, *((uuid_t *) &rc->config_hash_id)); + if(ctr->react_action == RRDCALC_REACT_NEW) { + if(ctr->rrdset) + rrdcalc_link_to_rrdset(ctr->rrdset, rc); - newrc->dimensions = string_strdupz(dimension); - newrc->foreachdim = NULL; - rc->foreachcounter++; - newrc->foreachcounter = rc->foreachcounter; - - newrc->green = rc->green; - newrc->red = rc->red; - newrc->value = NAN; - newrc->old_value = NAN; - - newrc->delay_up_duration = rc->delay_up_duration; - newrc->delay_down_duration = rc->delay_down_duration; - newrc->delay_max_duration = rc->delay_max_duration; - newrc->delay_multiplier = rc->delay_multiplier; - - newrc->last_repeat = 0; - newrc->times_repeat = 0; - newrc->warn_repeat_every = rc->warn_repeat_every; - newrc->crit_repeat_every = rc->crit_repeat_every; - - newrc->group = rc->group; - newrc->after = rc->after; - newrc->before = rc->before; - newrc->update_every = rc->update_every; - newrc->options = rc->options; - - newrc->exec = string_dup(rc->exec); - newrc->recipient = string_dup(rc->recipient); - newrc->source = string_dup(rc->source); - newrc->units = string_dup(rc->units); - newrc->info = string_dup(rc->info); - newrc->original_info = string_dup(rc->original_info); - - newrc->classification = string_dup(rc->classification); - newrc->component = string_dup(rc->component); - newrc->type = string_dup(rc->type); - - if(rc->calculation) { - newrc->calculation = expression_parse(rc->calculation->source, NULL, NULL); - if(!newrc->calculation) - error("Health alarm '%s.%s': failed to parse calculation expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->calculation->source); + else if (ctr->from_rrdcalctemplate) + rrdcontext_foreach_instance_with_rrdset_in_context(host, string2str(ctr->from_rrdcalctemplate->context), rrdcalc_check_and_link_rrdset_callback, rc); } - - if(rc->warning) { - newrc->warning = expression_parse(rc->warning->source, NULL, NULL); - if(!newrc->warning) - error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->warning->source); - } - - if(rc->critical) { - newrc->critical = expression_parse(rc->critical->source, NULL, NULL); - if(!newrc->critical) - error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->critical->source); - } - - return newrc; } -void rrdcalc_free(RRDCALC *rc) { +// ---------------------------------------------------------------------------- +// RRDCALC rrdhost index management - destructor + +static void rrdcalc_free_internals(RRDCALC *rc) { if(unlikely(!rc)) return; expression_free(rc->calculation); expression_free(rc->warning); expression_free(rc->critical); + string_freez(rc->key); string_freez(rc->name); string_freez(rc->chart); string_freez(rc->dimensions); - string_freez(rc->foreachdim); + string_freez(rc->foreach_dimension); string_freez(rc->exec); string_freez(rc->recipient); string_freez(rc->source); @@ -635,42 +583,138 @@ void rrdcalc_free(RRDCALC *rc) { string_freez(rc->module_match); string_freez(rc->plugin_match); - simple_pattern_free(rc->spdim); + simple_pattern_free(rc->foreach_dimension_pattern); simple_pattern_free(rc->host_labels_pattern); simple_pattern_free(rc->module_pattern); simple_pattern_free(rc->plugin_pattern); - - freez(rc->family); - freez(rc); } -void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) { - if(unlikely(!rc)) return; +static void rrdcalc_rrdhost_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *rrdhost __maybe_unused) { + RRDCALC *rc = rrdcalc; + //RRDHOST *host = rrdhost; - debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdhost_hostname(host)); + if(unlikely(rc->rrdset)) + rrdcalc_unlink_from_rrdset(rc, false); - // unlink it from RRDSET - if(rc->rrdset) rrdsetcalc_unlink(rc); + // any destruction actions that require other locks + // have to be placed in rrdcalc_del(), because the object is actually locked for deletion - // unlink it from RRDHOST - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->host_alarms, rc, prev, next); - - rrdcalc_free(rc); + rrdcalc_free_internals(rc); } -void rrdcalc_foreach_unlink_and_free(RRDHOST *host, RRDCALC *rc) { - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->alarms_with_foreach, rc, prev, next); - rrdcalc_free(rc); +// ---------------------------------------------------------------------------- +// RRDCALC rrdhost index management - index API + +void rrdcalc_rrdhost_index_init(RRDHOST *host) { + if(!host->rrdcalc_root_index) { + host->rrdcalc_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_insert_callback, NULL); + dictionary_register_react_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_react_callback, NULL); + dictionary_register_delete_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_delete_callback, host); + } } -static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) { - for(RRDCALC *rc = alarms ; rc ; ) { - RRDCALC *rc_next = rc->next; +void rrdcalc_rrdhost_index_destroy(RRDHOST *host) { + dictionary_destroy(host->rrdcalc_root_index); + host->rrdcalc_root_index = NULL; +} - if (!rc->host_labels) { - rc = rc_next; - continue; +void rrdcalc_add_from_rrdcalctemplate(RRDHOST *host, RRDCALCTEMPLATE *rt, RRDSET *st, const char *overwrite_alert_name, const char *overwrite_dimensions) { + char key[RRDCALC_MAX_KEY_SIZE + 1]; + size_t key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_id(st), + overwrite_alert_name?overwrite_alert_name:string2str(rt->name)); + + struct rrdcalc_constructor tmp = { + .rrdhost = host, + .from_config = NULL, + .from_rrdcalctemplate = rt, + .rrdset = st, + .overwrite_alert_name = overwrite_alert_name, + .overwrite_dimensions = overwrite_dimensions, + .react_action = RRDCALC_REACT_NONE, + }; + + dictionary_set_advanced(host->rrdcalc_root_index, key, (ssize_t)(key_len + 1), NULL, sizeof(RRDCALC), &tmp); + if(tmp.react_action != RRDCALC_REACT_NEW) + error("RRDCALC: from template '%s' on chart '%s' with key '%s', failed to be added to host '%s'. It already exists.", + string2str(rt->name), rrdset_id(st), key, rrdhost_hostname(host)); +} + +int rrdcalc_add_from_config(RRDHOST *host, RRDCALC *rc) { + if(!rc->chart) { + error("Health configuration for alarm '%s' does not have a chart", rrdcalc_name(rc)); + return 0; + } + + if(!rc->update_every) { + error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rrdcalc_chart_name(rc), rrdcalc_name(rc)); + return 0; + } + + if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->calculation && !rc->warning && !rc->critical) { + error("Health configuration for alarm '%s.%s' is useless (no db lookup, no calculation, no warning and no critical expressions)", rrdcalc_chart_name(rc), rrdcalc_name(rc)); + return 0; + } + + char key[RRDCALC_MAX_KEY_SIZE + 1]; + size_t key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, string2str(rc->chart), string2str(rc->name)); + + struct rrdcalc_constructor tmp = { + .rrdhost = host, + .from_config = rc, + .from_rrdcalctemplate = NULL, + .rrdset = NULL, + .react_action = RRDCALC_REACT_NONE, + }; + + int ret = 1; + RRDCALC *t = dictionary_set_advanced(host->rrdcalc_root_index, key, (ssize_t)(key_len + 1), rc, sizeof(RRDCALC), &tmp); + if(tmp.react_action == RRDCALC_REACT_NEW) { + // we copied rc into the dictionary, so we have to free the container here + freez(rc); + rc = t; + + // since we loaded this config from configuration, we need to check if we can link it to alarms + RRDSET *st; + rrdset_foreach_read(st, host) { + if (unlikely(rrdcalc_check_and_link_rrdset_callback(st, rc) == -1)) + break; } + rrdset_foreach_done(st); + } + else { + error( + "RRDCALC: from config '%s' on chart '%s' failed to be added to host '%s'. It already exists.", + string2str(rc->name), + string2str(rc->chart), + rrdhost_hostname(host)); + + ret = 0; + + // free all of it, internals and the container + rrdcalc_free_unused_rrdcalc_loaded_from_config(rc); + } + + return ret; +} + +static void rrdcalc_unlink_and_delete(RRDHOST *host, RRDCALC *rc, bool having_ll_wrlock) { + if(rc->rrdset) + rrdcalc_unlink_from_rrdset(rc, having_ll_wrlock); + + dictionary_del_advanced(host->rrdcalc_root_index, string2str(rc->key), (ssize_t)string_strlen(rc->key) + 1); +} + + +// ---------------------------------------------------------------------------- +// RRDCALC cleanup API functions + +void rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(RRDHOST *host) { + RRDCALC *rc; + foreach_rrdcalc_in_rrdhost_reentrant(host, rc) { + if (!rc->host_labels) + continue; if(!rrdlabels_match_simple_pattern_parsed(host->rrdlabels, rc->host_labels_pattern, '=')) { info("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'", @@ -678,21 +722,13 @@ static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) { rrdhost_hostname(host), rrdcalc_host_labels(rc)); - if(host->host_alarms == alarms) - rrdcalc_unlink_and_free(host, rc); - else - rrdcalc_foreach_unlink_and_free(host, rc); + rrdcalc_unlink_and_delete(host, rc, false); } - rc = rc_next; } + foreach_rrdcalc_in_rrdhost_done(rc); } -void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host) { - rrdcalc_labels_unlink_alarm_loop(host, host->host_alarms); - rrdcalc_labels_unlink_alarm_loop(host, host->alarms_with_foreach); -} - -void rrdcalc_labels_unlink() { +void rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts() { rrd_rdlock(); RRDHOST *host; @@ -700,14 +736,45 @@ void rrdcalc_labels_unlink() { if (unlikely(!host->health_enabled)) continue; - if (host->rrdlabels) { - rrdhost_wrlock(host); - - rrdcalc_labels_unlink_alarm_from_host(host); - - rrdhost_unlock(host); - } + if (host->rrdlabels) + rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host); } rrd_unlock(); } + +void rrdcalc_unlink_all_rrdset_alerts(RRDSET *st) { + RRDCALC *rc, *last = NULL; + netdata_rwlock_wrlock(&st->alerts.rwlock); + while((rc = st->alerts.base)) { + if(last == rc) { + error("RRDCALC: malformed list of alerts linked to chart - cannot cleanup - giving up."); + break; + } + last = rc; + + if(rc->run_flags & RRDCALC_FLAG_FROM_TEMPLATE) { + // if the alert comes from a template we can just delete it + rrdcalc_unlink_and_delete(st->rrdhost, rc, true); + } + else { + // this is a configuration for a specific chart + // it should stay in the list + rrdcalc_unlink_from_rrdset(rc, true); + } + + } + netdata_rwlock_unlock(&st->alerts.rwlock); +} + +void rrdcalc_delete_all(RRDHOST *host) { + dictionary_flush(host->rrdcalc_root_index); +} + +void rrdcalc_free_unused_rrdcalc_loaded_from_config(RRDCALC *rc) { + if(rc->rrdset) + rrdcalc_unlink_from_rrdset(rc, false); + + rrdcalc_free_internals(rc); + freez(rc); +} diff --git a/database/rrdcalc.h b/database/rrdcalc.h index 02ccaefa39..e9f53453b7 100644 --- a/database/rrdcalc.h +++ b/database/rrdcalc.h @@ -10,28 +10,35 @@ // (defined in their update_every member below) // They increase the overhead of netdata. // -// These calculations are allocated and linked (->next) -// under RRDHOST. -// Then are also linked to RRDSET (of course only when the -// chart is found, via ->rrdset_next and ->rrdset_prev). -// This double-linked list is maintained sorted at all times -// having as RRDSET.calculations the RRDCALC to be processed -// next. +// These calculations are stored under RRDHOST. +// Then are also linked to RRDSET (of course only when a +// matching chart is found). -#define RRDCALC_FLAG_DB_ERROR 0x00000001 -#define RRDCALC_FLAG_DB_NAN 0x00000002 -/* #define RRDCALC_FLAG_DB_STALE 0x00000004 */ -#define RRDCALC_FLAG_CALC_ERROR 0x00000008 -#define RRDCALC_FLAG_WARN_ERROR 0x00000010 -#define RRDCALC_FLAG_CRIT_ERROR 0x00000020 -#define RRDCALC_FLAG_RUNNABLE 0x00000040 -#define RRDCALC_FLAG_DISABLED 0x00000080 -#define RRDCALC_FLAG_SILENCED 0x00000100 -#define RRDCALC_FLAG_RUN_ONCE 0x00000200 -#define RRDCALC_FLAG_NO_CLEAR_NOTIFICATION 0x80000000 +typedef enum { + RRDCALC_FLAG_DB_ERROR = (1 << 0), + RRDCALC_FLAG_DB_NAN = (1 << 1), + // RRDCALC_FLAG_DB_STALE = (1 << 2), + RRDCALC_FLAG_CALC_ERROR = (1 << 3), + RRDCALC_FLAG_WARN_ERROR = (1 << 4), + RRDCALC_FLAG_CRIT_ERROR = (1 << 5), + RRDCALC_FLAG_RUNNABLE = (1 << 6), + RRDCALC_FLAG_DISABLED = (1 << 7), + RRDCALC_FLAG_SILENCED = (1 << 8), + RRDCALC_FLAG_RUN_ONCE = (1 << 9), + RRDCALC_FLAG_FROM_TEMPLATE = (1 << 10), // the rrdcalc has been created from a template +} RRDCALC_FLAGS; +typedef enum { + // This list uses several other options from RRDR_OPTIONS for db lookups. + // To add an item here, you need to reserve a bit in RRDR_OPTIONS. + RRDCALC_OPTION_NO_CLEAR_NOTIFICATION = 0x80000000, +} RRDCALC_OPTIONS; + +#define RRDCALC_ALL_OPTIONS_EXCLUDING_THE_RRDR_ONES (RRDCALC_OPTION_NO_CLEAR_NOTIFICATION) struct rrdcalc { + STRING *key; // the unique key in the host's rrdcalc_root_index + uint32_t id; // the unique id of this alarm uint32_t next_event_id; // the next event id that will be used for this alarm @@ -68,14 +75,12 @@ struct rrdcalc { // database lookup settings STRING *dimensions; // the chart dimensions - STRING *foreachdim; // the group of dimensions that the `foreach` will be applied. - SIMPLE_PATTERN *spdim; // used if and only if there is a simple pattern for the chart. - int foreachcounter; // the number of alarms created with foreachdim, this also works as an id of the - // children + STRING *foreach_dimension; // the group of dimensions that the `foreach` will be applied. + SIMPLE_PATTERN *foreach_dimension_pattern; // used if and only if there is a simple pattern for the chart. RRDR_GROUPING group; // grouping method: average, max, etc. int before; // ending point in time-series int after; // starting point in time-series - uint32_t options; // calculation options + RRDCALC_OPTIONS options; // configuration options // ------------------------------------------------------------------------ // expressions related to the alarm @@ -113,7 +118,7 @@ struct rrdcalc { NETDATA_DOUBLE value; // the current value of the alarm NETDATA_DOUBLE old_value; // the previous value of the alarm - uint32_t rrdcalc_flags; // check RRDCALC_FLAG_* + RRDCALC_FLAGS run_flags; // check RRDCALC_FLAG_* time_t last_updated; // the last update timestamp of the alarm time_t next_update; // the next update timestamp of the alarm @@ -132,20 +137,17 @@ struct rrdcalc { // ------------------------------------------------------------------------ // variables this alarm exposes to the rest of the alarms - RRDVAR *local; - RRDVAR *family; - RRDVAR *hostid; - RRDVAR *hostname; + const RRDVAR_ACQUIRED *rrdvar_local; + const RRDVAR_ACQUIRED *rrdvar_family; + const RRDVAR_ACQUIRED *rrdvar_host_chart_id; + const RRDVAR_ACQUIRED *rrdvar_host_chart_name; // ------------------------------------------------------------------------ // the chart this alarm it is linked to + size_t labels_version; struct rrdset *rrdset; - // linking of this alarm on its chart - struct rrdcalc *rrdset_next; - struct rrdcalc *rrdset_prev; - struct rrdcalc *next; struct rrdcalc *prev; }; @@ -164,14 +166,17 @@ struct rrdcalc { #define rrdcalc_original_info(rc) string2str((rc)->original_info) #define rrdcalc_info(rc) string2str((rc)->info) #define rrdcalc_dimensions(rc) string2str((rc)->dimensions) -#define rrdcalc_foreachdim(rc) string2str((rc)->foreachdim) +#define rrdcalc_foreachdim(rc) string2str((rc)->foreach_dimension) #define rrdcalc_host_labels(rc) string2str((rc)->host_labels) -#define foreach_rrdcalc_in_rrdset(st, rc) \ - DOUBLE_LINKED_LIST_FOREACH_FORWARD((st)->alarms, rc, rrdset_prev, rrdset_next) +#define foreach_rrdcalc_in_rrdhost_read(host, rc) \ + dfe_start_read((host)->rrdcalc_root_index, rc) \ -#define foreach_rrdcalc_in_rrdhost(host, rc) \ - DOUBLE_LINKED_LIST_FOREACH_FORWARD((host)->host_alarms, rc, prev, next) +#define foreach_rrdcalc_in_rrdhost_reentrant(host, rc) \ + dfe_start_reentrant((host)->rrdcalc_root_index, rc) + +#define foreach_rrdcalc_in_rrdhost_done(rc) \ + dfe_done(rc) struct alert_config { STRING *alarm; @@ -213,26 +218,24 @@ struct alert_config { #define RRDCALC_HAS_DB_LOOKUP(rc) ((rc)->after) -extern void rrdsetcalc_link_matching(RRDSET *st); -extern void rrdsetcalc_unlink(RRDCALC *rc); -extern RRDCALC *rrdcalc_find(RRDSET *st, const char *name); +extern void rrdcalc_update_info_using_rrdset_labels(RRDCALC *rc); + +extern void rrdcalc_link_matching_alerts_to_rrdset(RRDSET *st); + +extern const RRDCALC_ACQUIRED *rrdcalc_from_rrdset_get(RRDSET *st, const char *alert_name); +extern void rrdcalc_from_rrdset_release(RRDSET *st, const RRDCALC_ACQUIRED *rca); +extern RRDCALC *rrdcalc_acquired_to_rrdcalc(const RRDCALC_ACQUIRED *rca); extern const char *rrdcalc_status2string(RRDCALC_STATUS status); -extern void rrdcalc_free(RRDCALC *rc); -extern void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc); +extern void rrdcalc_free_unused_rrdcalc_loaded_from_config(RRDCALC *rc); -extern int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name); extern uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id); -extern RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart); -extern RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const char *name, const char *dimension); -extern void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc); -extern void dimension_remove_pipe_comma(char *str); -extern char *alarm_name_with_dim(const char *name, size_t namelen, const char *dim, size_t dimlen); -extern void rrdcalc_update_rrdlabels(RRDSET *st); +extern void rrdcalc_add_from_rrdcalctemplate(RRDHOST *host, RRDCALCTEMPLATE *rt, RRDSET *st, const char *overwrite_alert_name, const char *overwrite_dimensions); +extern int rrdcalc_add_from_config(RRDHOST *host, RRDCALC *rc); -extern void rrdcalc_labels_unlink(); -extern void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host); +extern void rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts(); +extern void rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(RRDHOST *host); static inline int rrdcalc_isrepeating(RRDCALC *rc) { if (unlikely(rc->warn_repeat_every > 0 || rc->crit_repeat_every > 0)) { @@ -241,6 +244,12 @@ static inline int rrdcalc_isrepeating(RRDCALC *rc) { return 0; } +extern void rrdcalc_unlink_all_rrdset_alerts(RRDSET *st); +extern void rrdcalc_delete_all(RRDHOST *host); + +extern void rrdcalc_rrdhost_index_init(RRDHOST *host); +extern void rrdcalc_rrdhost_index_destroy(RRDHOST *host); + #define RRDCALC_VAR_MAX 100 #define RRDCALC_VAR_FAMILY "$family" #define RRDCALC_VAR_LABEL "$label:" diff --git a/database/rrdcalctemplate.c b/database/rrdcalctemplate.c index 0db76d6c04..87e085c932 100644 --- a/database/rrdcalctemplate.c +++ b/database/rrdcalctemplate.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#define NETDATA_HEALTH_INTERNALS #include "rrd.h" // ---------------------------------------------------------------------------- @@ -11,45 +10,84 @@ * @param rt is the template used to create the chart. * @param st is the chart where the alarm will be attached. */ -void rrdcalctemplate_check_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) { - if(rt->context != st->context) - return; - if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, rrdset_name(st))) - return; +static char *rrdcalc_alert_name_with_dimension(const char *name, size_t namelen, const char *dim, size_t dimlen) { + char *newname,*move; + + newname = mallocz(namelen + dimlen + 2); + move = newname; + memcpy(move, name, namelen); + move += namelen; + + *move++ = '_'; + memcpy(move, dim, dimlen); + move += dimlen; + *move = '\0'; + + return newname; +} + +bool rrdcalctemplate_check_rrdset_conditions(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) { + if(rt->context != st->context) + return false; + + if(rt->foreach_dimension_pattern && !rrdset_number_of_dimensions(st)) + return false; + + if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, rrdset_name(st)) && !simple_pattern_matches(rt->charts_pattern, rrdset_id(st))) + return false; if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, rrdset_family(st))) - return; + return false; if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, rrdset_module_name(st))) - return; + return false; if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, rrdset_plugin_name(st))) - return; + return false; if(host->rrdlabels && rt->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(host->rrdlabels, rt->host_labels_pattern, '=')) + return false; + + return true; +} + +void rrdcalctemplate_check_rrddim_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDDIM *rd, RRDHOST *host) { + if (simple_pattern_matches(rt->foreach_dimension_pattern, rrddim_id(rd)) || simple_pattern_matches(rt->foreach_dimension_pattern, rrddim_name(rd))) { + char *overwrite_alert_name = rrdcalc_alert_name_with_dimension( + rrdcalctemplate_name(rt), string_strlen(rt->name), rrddim_name(rd), string_strlen(rd->name)); + rrdcalc_add_from_rrdcalctemplate(host, rt, st, overwrite_alert_name, rrddim_name(rd)); + freez(overwrite_alert_name); + } +} + +void rrdcalctemplate_check_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) { + if(!rrdcalctemplate_check_rrdset_conditions(rt, st, host)) return; - RRDCALC *rc = rrdcalc_create_from_template(host, rt, rrdset_id(st)); - if (unlikely(!rc)) - info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rrdcalctemplate_name(rt), rrdset_id(st), rrdhost_hostname(host)); -#ifdef NETDATA_INTERNAL_CHECKS - else if (rc->rrdset != st && !rc->foreachdim) //When we have a template with foreachdim, the child will be added to the index late - error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st)); -#endif + if(!rt->foreach_dimension_pattern) { + rrdcalc_add_from_rrdcalctemplate(host, rt, st, NULL, NULL); + return; + } + + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + rrdcalctemplate_check_rrddim_conditions_and_link(rt, st, rd, host); + } + rrddim_foreach_done(rd); } -void rrdcalctemplate_link_matching(RRDSET *st) { +void rrdcalctemplate_link_matching_templates_to_rrdset(RRDSET *st) { RRDHOST *host = st->rrdhost; - RRDCALCTEMPLATE *rt; - foreach_rrdcalctemplate_in_rrdhost(host, rt) + RRDCALCTEMPLATE *rt; + foreach_rrdcalctemplate_read(host, rt) { rrdcalctemplate_check_conditions_and_link(rt, st, host); + } + foreach_rrdcalctemplate_done(rt); } -inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) { - if(unlikely(!rt)) return; - +static void rrdcalctemplate_free_internals(RRDCALCTEMPLATE *rt) { expression_free(rt->calculation); expression_free(rt->warning); expression_free(rt->critical); @@ -77,19 +115,127 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) { string_freez(rt->units); string_freez(rt->info); string_freez(rt->dimensions); - string_freez(rt->foreachdim); + string_freez(rt->foreach_dimension); string_freez(rt->host_labels); - simple_pattern_free(rt->spdim); + simple_pattern_free(rt->foreach_dimension_pattern); simple_pattern_free(rt->host_labels_pattern); - freez(rt); } -inline void rrdcalctemplate_unlink_and_free(RRDHOST *host, RRDCALCTEMPLATE *rt) { +void rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(RRDCALCTEMPLATE *rt) { if(unlikely(!rt)) return; - debug(D_HEALTH, "Health removing template '%s' of host '%s'", rrdcalctemplate_name(rt), rrdhost_hostname(host)); - - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->alarms_templates, rt, prev, next); - - rrdcalctemplate_free(rt); + rrdcalctemplate_free_internals(rt); + freez(rt); +} +static void rrdcalctemplate_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalctemplate, void *added_bool) { + RRDCALCTEMPLATE *rt = rrdcalctemplate; (void)rt; + + bool *added = added_bool; + *added = true; + + debug(D_HEALTH, "Health configuration adding template '%s'" + ": context '%s'" + ", exec '%s'" + ", recipient '%s'" + ", green " NETDATA_DOUBLE_FORMAT_AUTO + ", red " NETDATA_DOUBLE_FORMAT_AUTO + ", lookup: group %d" + ", after %d" + ", before %d" + ", options %u" + ", dimensions '%s'" + ", for each dimension '%s'" + ", update every %d" + ", calculation '%s'" + ", warning '%s'" + ", critical '%s'" + ", source '%s'" + ", delay up %d" + ", delay down %d" + ", delay max %d" + ", delay_multiplier %f" + ", warn_repeat_every %u" + ", crit_repeat_every %u", + rrdcalctemplate_name(rt), + (rt->context)?string2str(rt->context):"NONE", + (rt->exec)?rrdcalctemplate_exec(rt):"DEFAULT", + (rt->recipient)?rrdcalctemplate_recipient(rt):"DEFAULT", + rt->green, + rt->red, + (int)rt->group, + rt->after, + rt->before, + rt->options, + (rt->dimensions)?rrdcalctemplate_dimensions(rt):"NONE", + (rt->foreach_dimension)?rrdcalctemplate_foreachdim(rt):"NONE", + rt->update_every, + (rt->calculation)?rt->calculation->parsed_as:"NONE", + (rt->warning)?rt->warning->parsed_as:"NONE", + (rt->critical)?rt->critical->parsed_as:"NONE", + rrdcalctemplate_source(rt), + rt->delay_up_duration, + rt->delay_down_duration, + rt->delay_max_duration, + rt->delay_multiplier, + rt->warn_repeat_every, + rt->crit_repeat_every + ); +} + +static void rrdcalctemplate_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalctemplate, void *rrdhost __maybe_unused) { + RRDCALCTEMPLATE *rt = rrdcalctemplate; + rrdcalctemplate_free_internals(rt); +} + +void rrdcalctemplate_index_init(RRDHOST *host) { + if(!host->rrdcalctemplate_root_index) { + host->rrdcalctemplate_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(host->rrdcalctemplate_root_index, rrdcalctemplate_insert_callback, NULL); + dictionary_register_delete_callback(host->rrdcalctemplate_root_index, rrdcalctemplate_delete_callback, host); + } +} + +void rrdcalctemplate_index_destroy(RRDHOST *host) { + dictionary_destroy(host->rrdcalctemplate_root_index); + host->rrdcalctemplate_root_index = NULL; +} + +inline void rrdcalctemplate_delete_all(RRDHOST *host) { + dictionary_flush(host->rrdcalctemplate_root_index); +} + +#define RRDCALCTEMPLATE_MAX_KEY_SIZE 1024 +static size_t rrdcalctemplate_key(char *dst, size_t dst_len, const char *name, const char *family_match) { + return snprintfz(dst, dst_len, "%s/%s", name, (family_match && *family_match)?family_match:"*"); +} + +void rrdcalctemplate_add_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) { + if(unlikely(!rt->context)) { + error("Health configuration for template '%s' does not have a context", rrdcalctemplate_name(rt)); + return; + } + + if(unlikely(!rt->update_every)) { + error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rrdcalctemplate_name(rt)); + return; + } + + if(unlikely(!RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) && !rt->calculation && !rt->warning && !rt->critical)) { + error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rrdcalctemplate_name(rt)); + return; + } + + char key[RRDCALCTEMPLATE_MAX_KEY_SIZE + 1]; + size_t key_len = rrdcalctemplate_key(key, RRDCALCTEMPLATE_MAX_KEY_SIZE, rrdcalctemplate_name(rt), rrdcalctemplate_family_match(rt)); + + bool added = false; + dictionary_set_advanced(host->rrdcalctemplate_root_index, key, (ssize_t)(key_len + 1), rt, sizeof(*rt), &added); + + if(added) + freez(rt); + else { + info("Health configuration template '%s' already exists for host '%s'.", rrdcalctemplate_name(rt), rrdhost_hostname(host)); + rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(rt); + } } diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h index cbb23df4d3..9b496c63e1 100644 --- a/database/rrdcalctemplate.h +++ b/database/rrdcalctemplate.h @@ -48,14 +48,12 @@ struct rrdcalctemplate { // database lookup settings STRING *dimensions; // the chart dimensions - STRING *foreachdim; // the group of dimensions that the lookup will be applied. - SIMPLE_PATTERN *spdim; // used if and only if there is a simple pattern for the chart. - int foreachcounter; // the number of alarms created with foreachdim, this also works as an id of the - // children + STRING *foreach_dimension; // the group of dimensions that the lookup will be applied. + SIMPLE_PATTERN *foreach_dimension_pattern; // used if and only if there is a simple pattern for the chart. RRDR_GROUPING group; // grouping method: average, max, etc. int before; // ending point in time-series int after; // starting point in time-series - uint32_t options; // calculation options + RRDCALC_OPTIONS options; // configuration options // ------------------------------------------------------------------------ // notification delay settings @@ -87,8 +85,11 @@ struct rrdcalctemplate { struct rrdcalctemplate *prev; }; -#define foreach_rrdcalctemplate_in_rrdhost(host, rt) \ - DOUBLE_LINKED_LIST_FOREACH_FORWARD((host)->alarms_templates, rt, prev, next) +#define foreach_rrdcalctemplate_read(host, rt) \ + dfe_start_read((host)->rrdcalctemplate_root_index, rt) + +#define foreach_rrdcalctemplate_done(rt) \ + dfe_done(rt) #define rrdcalctemplate_name(rt) string2str((rt)->name) #define rrdcalctemplate_exec(rt) string2str((rt)->exec) @@ -104,14 +105,24 @@ struct rrdcalctemplate { #define rrdcalctemplate_info(rt) string2str((rt)->info) #define rrdcalctemplate_source(rt) string2str((rt)->source) #define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions) -#define rrdcalctemplate_foreachdim(rt) string2str((rt)->foreachdim) +#define rrdcalctemplate_foreachdim(rt) string2str((rt)->foreach_dimension) #define rrdcalctemplate_host_labels(rt) string2str((rt)->host_labels) #define RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) ((rt)->after) -extern void rrdcalctemplate_link_matching(RRDSET *st); +extern void rrdcalctemplate_link_matching_templates_to_rrdset(RRDSET *st); + +extern void rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(RRDCALCTEMPLATE *rt); +extern void rrdcalctemplate_delete_all(RRDHOST *host); +extern void rrdcalctemplate_add_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt); + +extern void rrdcalctemplate_check_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host); + +extern bool rrdcalctemplate_check_rrdset_conditions(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host); +extern void rrdcalctemplate_check_rrddim_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDDIM *rd, RRDHOST *host); + + +extern void rrdcalctemplate_index_init(RRDHOST *host); +extern void rrdcalctemplate_index_destroy(RRDHOST *host); -extern void rrdcalctemplate_free(RRDCALCTEMPLATE *rt); -extern void rrdcalctemplate_unlink_and_free(RRDHOST *host, RRDCALCTEMPLATE *rt); -extern void rrdcalctemplate_create_alarms(RRDHOST *host, RRDCALCTEMPLATE *rt, RRDSET *st); #endif //NETDATA_RRDCALCTEMPLATE_H diff --git a/database/rrdcontext.c b/database/rrdcontext.c index 4a77fea1b4..d4f5d1d848 100644 --- a/database/rrdcontext.c +++ b/database/rrdcontext.c @@ -8,6 +8,8 @@ int rrdcontext_enabled = CONFIG_BOOLEAN_YES; +// #define LOG_POST_PROCESSING_QUEUE_INSERTIONS 1 + #define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000 #define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120 #define RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC (1000 * USEC_PER_MS) @@ -432,11 +434,11 @@ static void rrdmetric_free(RRDMETRIC *rm) { // called when this rrdmetric is inserted to the rrdmetrics dictionary of a rrdinstance // the constructor of the rrdmetric object -static void rrdmetric_insert_callback(const char *id __maybe_unused, void *value, void *data) { +static void rrdmetric_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance) { RRDMETRIC *rm = value; // link it to its parent - rm->ri = data; + rm->ri = rrdinstance; // remove flags that we need to figure out at runtime rm->flags = rm->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics @@ -447,7 +449,7 @@ static void rrdmetric_insert_callback(const char *id __maybe_unused, void *value // called when this rrdmetric is deleted from the rrdmetrics dictionary of a rrdinstance // the destructor of the rrdmetric object -static void rrdmetric_delete_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) { +static void rrdmetric_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance __maybe_unused) { RRDMETRIC *rm = value; internal_error(rm->rrddim, "RRDMETRIC: '%s' is freed but there is a RRDDIM linked to it.", string2str(rm->id)); @@ -458,7 +460,7 @@ static void rrdmetric_delete_callback(const char *id __maybe_unused, void *value // called when the same rrdmetric is inserted again to the rrdmetrics dictionary of a rrdinstance // while this is called, the dictionary is write locked, but there may be other users of the object -static void rrdmetric_conflict_callback(const char *id __maybe_unused, void *oldv, void *newv, void *data __maybe_unused) { +static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *oldv, void *newv, void *rrdinstance __maybe_unused) { RRDMETRIC *rm = oldv; RRDMETRIC *rm_new = newv; @@ -518,11 +520,12 @@ static void rrdmetric_conflict_callback(const char *id __maybe_unused, void *old rrdmetric_free(rm_new); // the react callback will continue from here + return rrd_flag_is_updated(rm); } // this is called after the insert or the conflict callbacks, // but the dictionary is now unlocked -static void rrdmetric_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) { +static void rrdmetric_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance __maybe_unused) { RRDMETRIC *rm = value; rrdmetric_trigger_updates(rm, __FUNCTION__ ); } @@ -531,11 +534,11 @@ static void rrdmetrics_create_in_rrdinstance(RRDINSTANCE *ri) { if(unlikely(!ri)) return; if(likely(ri->rrdmetrics)) return; - ri->rrdmetrics = dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); - dictionary_register_insert_callback(ri->rrdmetrics, rrdmetric_insert_callback, (void *)ri); - dictionary_register_delete_callback(ri->rrdmetrics, rrdmetric_delete_callback, (void *)ri); - dictionary_register_conflict_callback(ri->rrdmetrics, rrdmetric_conflict_callback, (void *)ri); - dictionary_register_react_callback(ri->rrdmetrics, rrdmetric_react_callback, (void *)ri); + ri->rrdmetrics = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_insert_callback(ri->rrdmetrics, rrdmetric_insert_callback, ri); + dictionary_register_delete_callback(ri->rrdmetrics, rrdmetric_delete_callback, ri); + dictionary_register_conflict_callback(ri->rrdmetrics, rrdmetric_conflict_callback, ri); + dictionary_register_react_callback(ri->rrdmetrics, rrdmetric_react_callback, ri); } static void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri) { @@ -669,7 +672,7 @@ static void rrdinstance_free(RRDINSTANCE *ri) { ri->rrdset = NULL; } -static void rrdinstance_insert_callback(const char *id __maybe_unused, void *value, void *data) { +static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext) { static STRING *ml_anomaly_rates_id = NULL; if(unlikely(!ml_anomaly_rates_id)) @@ -678,7 +681,7 @@ static void rrdinstance_insert_callback(const char *id __maybe_unused, void *val RRDINSTANCE *ri = value; // link it to its parent - ri->rc = data; + ri->rc = rrdcontext; ri->flags = ri->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics @@ -711,9 +714,7 @@ static void rrdinstance_insert_callback(const char *id __maybe_unused, void *val rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_NEW_OBJECT); } -static void rrdinstance_delete_callback(const char *id, void *value, void *data) { - (void)id; - RRDCONTEXT *rc = data; (void)rc; +static void rrdinstance_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext __maybe_unused) { RRDINSTANCE *ri = (RRDINSTANCE *)value; internal_error(ri->rrdset, "RRDINSTANCE: '%s' is freed but there is a RRDSET linked to it.", string2str(ri->id)); @@ -721,7 +722,7 @@ static void rrdinstance_delete_callback(const char *id, void *value, void *data) rrdinstance_free(ri); } -static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *oldv, void *newv, void *data __maybe_unused) { +static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *oldv, void *newv, void *rrdcontext __maybe_unused) { RRDINSTANCE *ri = (RRDINSTANCE *)oldv; RRDINSTANCE *ri_new = (RRDINSTANCE *)newv; @@ -739,10 +740,10 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING); } - if(ri->rrdset && ri->rrdset->chart_uuid && uuid_compare(ri->uuid, *ri->rrdset->chart_uuid) != 0) { + if(ri->rrdset && uuid_compare(ri->uuid, ri->rrdset->chart_uuid) != 0) { char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; uuid_unparse(ri->uuid, uuid1); - uuid_unparse(*ri->rrdset->chart_uuid, uuid2); + uuid_unparse(ri->rrdset->chart_uuid, uuid2); internal_error(true, "RRDINSTANCE: '%s' is linked to RRDSET '%s' but they have different UUIDs. RRDINSTANCE has '%s', RRDSET has '%s'", string2str(ri->id), rrdset_id(ri->rrdset), uuid1, uuid2); } @@ -823,9 +824,10 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o rrdinstance_free(ri_new); // the react callback will continue from here + return rrd_flag_is_updated(ri); } -static void rrdinstance_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) { +static void rrdinstance_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext __maybe_unused) { RRDINSTANCE *ri = value; rrdinstance_trigger_updates(ri, __FUNCTION__ ); @@ -837,11 +839,11 @@ void rrdinstances_create_in_rrdcontext(RRDCONTEXT *rc) { if(unlikely(!rc || rc->rrdinstances)) return; - rc->rrdinstances = dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); - dictionary_register_insert_callback(rc->rrdinstances, rrdinstance_insert_callback, (void *)rc); - dictionary_register_delete_callback(rc->rrdinstances, rrdinstance_delete_callback, (void *)rc); - dictionary_register_conflict_callback(rc->rrdinstances, rrdinstance_conflict_callback, (void *)rc); - dictionary_register_react_callback(rc->rrdinstances, rrdinstance_react_callback, (void *)rc); + rc->rrdinstances = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_insert_callback(rc->rrdinstances, rrdinstance_insert_callback, rc); + dictionary_register_delete_callback(rc->rrdinstances, rrdinstance_delete_callback, rc); + dictionary_register_conflict_callback(rc->rrdinstances, rrdinstance_conflict_callback, rc); + dictionary_register_react_callback(rc->rrdinstances, rrdinstance_react_callback, rc); } void rrdinstances_destroy_from_rrdcontext(RRDCONTEXT *rc) { @@ -907,7 +909,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) { .flags = RRD_FLAG_NONE, // no need for atomics .rrdset = st, }; - uuid_copy(tri.uuid, *st->chart_uuid); + uuid_copy(tri.uuid, st->chart_uuid); RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, string2str(tri.id), &tri, sizeof(tri)); @@ -934,7 +936,6 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) { RRDINSTANCE *ri_old = rrdinstance_acquired_value(ria_old); // migrate all dimensions to the new metrics - rrdset_rdlock(st); RRDDIM *rd; rrddim_foreach_read(rd, st) { if (!rd->rrdmetric) continue; @@ -950,7 +951,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) { rrdmetric_from_rrddim(rd); } - rrdset_unlock(st); + rrddim_foreach_done(rd); // mark the old instance, ready to be deleted if(!rrd_flag_check(ri_old, RRD_FLAG_OWN_LABELS)) @@ -966,7 +967,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) { /* // trigger updates on the old context - if(!dictionary_stats_entries(rc_old->rrdinstances) && !dictionary_stats_referenced_items(rc_old->rrdinstances)) { + if(!dictionary_entries(rc_old->rrdinstances) && !dictionary_stats_referenced_items(rc_old->rrdinstances)) { rrdcontext_lock(rc_old); rc_old->flags = ((rc_old->flags & RRD_FLAG_QUEUED)?RRD_FLAG_QUEUED:RRD_FLAG_NONE)|RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION; rc_old->first_time_t = 0; @@ -1102,9 +1103,8 @@ static void rrdcontext_freez(RRDCONTEXT *rc) { string_freez(rc->family); } -static void rrdcontext_insert_callback(const char *id, void *value, void *data) { - (void)id; - RRDHOST *host = (RRDHOST *)data; +static void rrdcontext_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost) { + RRDHOST *host = (RRDHOST *)rrdhost; RRDCONTEXT *rc = (RRDCONTEXT *)value; rc->rrdhost = host; @@ -1167,10 +1167,7 @@ static void rrdcontext_insert_callback(const char *id, void *value, void *data) rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_NEW_OBJECT); } -static void rrdcontext_delete_callback(const char *id, void *value, void *data) { - (void)id; - RRDHOST *host = (RRDHOST *)data; - (void)host; +static void rrdcontext_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost __maybe_unused) { RRDCONTEXT *rc = (RRDCONTEXT *)value; @@ -1179,18 +1176,14 @@ static void rrdcontext_delete_callback(const char *id, void *value, void *data) rrdcontext_freez(rc); } -static void rrdcontext_conflict_callback(const char *id, void *oldv, void *newv, void *data) { - (void)id; - RRDHOST *host = (RRDHOST *)data; - (void)host; - +static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *oldv, void *newv, void *rrdhost __maybe_unused) { RRDCONTEXT *rc = (RRDCONTEXT *)oldv; RRDCONTEXT *rc_new = (RRDCONTEXT *)newv; //current rc is not archived, new_rc is archived, dont merge if (!rrd_flag_is_archived(rc) && rrd_flag_is_archived(rc_new)) { rrdcontext_freez(rc_new); - return; + return false; } rrdcontext_lock(rc); @@ -1246,11 +1239,11 @@ static void rrdcontext_conflict_callback(const char *id, void *oldv, void *newv, rrdcontext_freez(rc_new); // the react callback will continue from here + return rrd_flag_is_updated(rc); } -static void rrdcontext_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) { +static void rrdcontext_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdhost __maybe_unused) { RRDCONTEXT *rc = (RRDCONTEXT *)value; - rrdcontext_trigger_updates(rc, __FUNCTION__ ); } @@ -1259,33 +1252,35 @@ static void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function) { rrdcontext_queue_for_post_processing(rc, function, rc->flags); } -static void rrdcontext_hub_queue_insert_callback(const char *name __maybe_unused, void *context, void *data __maybe_unused) { +static void rrdcontext_hub_queue_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) { RRDCONTEXT *rc = context; rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB); rc->queue.queued_ut = now_realtime_usec(); rc->queue.queued_flags = rrd_flags_get(rc); } -static void rrdcontext_hub_queue_delete_callback(const char *name __maybe_unused, void *context, void *data __maybe_unused) { +static void rrdcontext_hub_queue_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) { RRDCONTEXT *rc = context; rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_HUB); } -static void rrdcontext_hub_queue_conflict_callback(const char *name __maybe_unused, void *context, void *new_context __maybe_unused, void *data __maybe_unused) { +static bool rrdcontext_hub_queue_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *new_context __maybe_unused, void *nothing __maybe_unused) { // context and new_context are the same // we just need to update the timings RRDCONTEXT *rc = context; rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB); rc->queue.queued_ut = now_realtime_usec(); rc->queue.queued_flags |= rrd_flags_get(rc); + + return true; } -static void rrdcontext_post_processing_queue_insert_callback(const char *name __maybe_unused, void *context, void *data __maybe_unused) { +static void rrdcontext_post_processing_queue_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) { RRDCONTEXT *rc = context; rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING); } -static void rrdcontext_post_processing_queue_delete_callback(const char *name __maybe_unused, void *context, void *data __maybe_unused) { +static void rrdcontext_post_processing_queue_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) { RRDCONTEXT *rc = context; rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING); } @@ -1297,23 +1292,19 @@ void rrdhost_create_rrdcontexts(RRDHOST *host) { if(unlikely(!host)) return; if(likely(host->rrdctx)) return; - host->rrdctx = (RRDCONTEXTS *)dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); - dictionary_register_insert_callback((DICTIONARY *)host->rrdctx, rrdcontext_insert_callback, (void *)host); - dictionary_register_delete_callback((DICTIONARY *)host->rrdctx, rrdcontext_delete_callback, (void *)host); - dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx, rrdcontext_conflict_callback, (void *)host); - dictionary_register_react_callback((DICTIONARY *)host->rrdctx, rrdcontext_react_callback, (void *)host); + host->rrdctx = (RRDCONTEXTS *)dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_insert_callback((DICTIONARY *)host->rrdctx, rrdcontext_insert_callback, host); + dictionary_register_delete_callback((DICTIONARY *)host->rrdctx, rrdcontext_delete_callback, host); + dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx, rrdcontext_conflict_callback, host); + dictionary_register_react_callback((DICTIONARY *)host->rrdctx, rrdcontext_react_callback, host); - host->rrdctx_hub_queue = (RRDCONTEXTS *)dictionary_create( - DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE); + host->rrdctx_hub_queue = (RRDCONTEXTS *)dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE); dictionary_register_insert_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_insert_callback, NULL); dictionary_register_delete_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_delete_callback, NULL); dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_conflict_callback, NULL); - host->rrdctx_post_processing_queue = (RRDCONTEXTS *)dictionary_create( - DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE); + host->rrdctx_post_processing_queue = (RRDCONTEXTS *)dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_VALUE_LINK_DONT_CLONE); dictionary_register_insert_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_post_processing_queue_insert_callback, NULL); dictionary_register_delete_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_post_processing_queue_delete_callback, NULL); @@ -1331,7 +1322,7 @@ void rrdhost_destroy_rrdcontexts(RRDHOST *host) { RRDCONTEXT *rc; dfe_start_write(old, rc) { - dictionary_del_having_write_lock(old, string2str(rc->id)); + dictionary_del(old, string2str(rc->id)); } dfe_done(rc); dictionary_destroy(old); @@ -1343,7 +1334,7 @@ void rrdhost_destroy_rrdcontexts(RRDHOST *host) { RRDCONTEXT *rc; dfe_start_write(old, rc) { - dictionary_del_having_write_lock(old, string2str(rc->id)); + dictionary_del(old, string2str(rc->id)); } dfe_done(rc); dictionary_destroy(old); @@ -1464,6 +1455,35 @@ void rrdcontext_db_rotation(void) { rrdcontext_next_db_rotation_ut = now_realtime_usec() + FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS * USEC_PER_SEC; } +int rrdcontext_foreach_instance_with_rrdset_in_context(RRDHOST *host, const char *context, int (*callback)(RRDSET *st, void *data), void *data) { + if(unlikely(!host || !context || !*context || !callback)) + return -1; + + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)host->rrdctx, context); + if(unlikely(!rca)) return -1; + + RRDCONTEXT *rc = rrdcontext_acquired_value(rca); + if(unlikely(!rc)) return -1; + + int ret = 0; + RRDINSTANCE *ri; + dfe_start_read(rc->rrdinstances, ri) { + if(ri->rrdset) { + int r = callback(ri->rrdset, data); + if(r >= 0) ret += r; + else { + ret = r; + break; + } + } + } + dfe_done(ri); + + rrdcontext_release(rca); + + return ret; +} + // ---------------------------------------------------------------------------- // ACLK interface @@ -1601,7 +1621,8 @@ struct rrdcontext_to_json { RRD_FLAGS combined_flags; }; -static inline int rrdmetric_to_json_callback(const char *id, void *value, void *data) { +static inline int rrdmetric_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *id = dictionary_acquired_item_name(item); struct rrdcontext_to_json * t = data; RRDMETRIC *rm = value; BUFFER *wb = t->wb; @@ -1673,7 +1694,9 @@ static inline int rrdmetric_to_json_callback(const char *id, void *value, void * return 1; } -static inline int rrdinstance_to_json_callback(const char *id, void *value, void *data) { +static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *id = dictionary_acquired_item_name(item); + struct rrdcontext_to_json *t_parent = data; RRDINSTANCE *ri = value; BUFFER *wb = t_parent->wb; @@ -1788,7 +1811,7 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void buffer_strcat(wb, "\""); } - if(options & RRDCONTEXT_OPTION_SHOW_LABELS && ri->rrdlabels && dictionary_stats_entries(ri->rrdlabels)) { + if(options & RRDCONTEXT_OPTION_SHOW_LABELS && ri->rrdlabels && dictionary_entries(ri->rrdlabels)) { buffer_sprintf(wb, ",\n\t\t\t\t\t\"labels\": {\n"); rrdlabels_to_buffer(ri->rrdlabels, wb, "\t\t\t\t\t\t", ":", "\"", ",\n", NULL, NULL, NULL, NULL); buffer_strcat(wb, "\n\t\t\t\t\t}"); @@ -1807,7 +1830,8 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void return 1; } -static inline int rrdcontext_to_json_callback(const char *id, void *value, void *data) { +static inline int rrdcontext_to_json_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *id = dictionary_acquired_item_name(item); struct rrdcontext_to_json *t_parent = data; RRDCONTEXT *rc = value; BUFFER *wb = t_parent->wb; @@ -1974,7 +1998,7 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R .written = 0, .now = now_realtime_sec(), }; - rrdcontext_to_json_callback(context, rc, &t_contexts); + rrdcontext_to_json_callback((DICTIONARY_ITEM *)rca, rc, &t_contexts); rrdcontext_release(rca); @@ -2300,10 +2324,10 @@ static inline bool rrdinstance_should_be_deleted(RRDINSTANCE *ri) { if(likely(ri->rrdset)) return false; - if(unlikely(dictionary_stats_referenced_items(ri->rrdmetrics) != 0)) + if(unlikely(dictionary_referenced_items(ri->rrdmetrics) != 0)) return false; - if(unlikely(dictionary_stats_entries(ri->rrdmetrics) != 0)) + if(unlikely(dictionary_entries(ri->rrdmetrics) != 0)) return false; if(ri->first_time_t || ri->last_time_t) @@ -2319,10 +2343,10 @@ static inline bool rrdcontext_should_be_deleted(RRDCONTEXT *rc) { if(likely(rrd_flag_check(rc, RRD_FLAGS_PREVENTING_DELETIONS))) return false; - if(unlikely(dictionary_stats_referenced_items(rc->rrdinstances) != 0)) + if(unlikely(dictionary_referenced_items(rc->rrdinstances) != 0)) return false; - if(unlikely(dictionary_stats_entries(rc->rrdinstances) != 0)) + if(unlikely(dictionary_entries(rc->rrdinstances) != 0)) return false; if(unlikely(rc->first_time_t || rc->last_time_t)) @@ -2364,7 +2388,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jo dfe_start_write(ri->rrdmetrics, rm) { if(rrdmetric_should_be_deleted(rm)) { if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE); - if(dictionary_del_having_write_lock(ri->rrdmetrics, string2str(rm->id)) != 0) + if(!dictionary_del(ri->rrdmetrics, string2str(rm->id))) error("RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.", string2str(rm->id), string2str(ri->id), @@ -2384,7 +2408,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jo if(rrdinstance_should_be_deleted(ri)) { if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE); - if(dictionary_del(rc->rrdinstances, string2str(ri->id)) != 0) + if(!dictionary_del(rc->rrdinstances, string2str(ri->id))) error("RRDCONTEXT: instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.", string2str(ri->id), string2str(rc->id), @@ -2405,7 +2429,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jo rrdcontext_dequeue_from_post_processing(rc); rrdcontext_delete_from_sql_unsafe(rc); - if(dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0) + if(!dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id))) error("RRDCONTEXT: context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.", string2str(rc->id), rrdhost_hostname(host)); @@ -2471,7 +2495,7 @@ static void rrdinstance_post_process_updates(RRDINSTANCE *ri, bool force, RRD_FL time_t min_first_time_t = LONG_MAX, max_last_time_t = 0; size_t metrics_active = 0, metrics_deleted = 0; bool live_retention = true, currently_collected = false; - if(dictionary_stats_entries(ri->rrdmetrics) > 0) { + if(dictionary_entries(ri->rrdmetrics) > 0) { RRDMETRIC *rm; dfe_start_read((DICTIONARY *)ri->rrdmetrics, rm) { if(unlikely(netdata_exit)) break; @@ -2574,7 +2598,7 @@ static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAG time_t min_first_time_t = LONG_MAX, max_last_time_t = 0; size_t instances_active = 0, instances_deleted = 0; bool live_retention = true, currently_collected = false, hidden = true; - if(dictionary_stats_entries(rc->rrdinstances) > 0) { + if(dictionary_entries(rc->rrdinstances) > 0) { RRDINSTANCE *ri; dfe_start_reentrant(rc->rrdinstances, ri) { if(unlikely(netdata_exit)) break; @@ -2714,7 +2738,7 @@ static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *fun rc, sizeof(*rc)); -#ifdef NETDATA_INTERNAL_CHECKS +#if(defined(NETDATA_INTERNAL_CHECKS) && defined(LOG_POST_PROCESSING_QUEUE_INSERTIONS)) { BUFFER *wb_flags = buffer_create(1000); rrd_flags_to_buffer(flags, wb_flags); @@ -2912,7 +2936,7 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now return; // check if there are queued items to send - if(!dictionary_stats_entries((DICTIONARY *)host->rrdctx_hub_queue)) + if(!dictionary_entries((DICTIONARY *)host->rrdctx_hub_queue)) return; if(!host->node_id) @@ -2975,7 +2999,7 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now rrdcontext_unlock(rc); // delete it from the master dictionary - if(dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0) + if(!dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id))) error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.", string2str(id), rrdhost_hostname(host)); @@ -3010,7 +3034,10 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now static void rrdcontext_main_cleanup(void *ptr) { struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; + // custom code + worker_unregister(); + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } @@ -3065,12 +3092,13 @@ void *rrdcontext_main(void *ptr) { worker_is_busy(WORKER_JOB_HOSTS); if(host->rrdctx_post_processing_queue) { - pp_queued_contexts_for_all_hosts += dictionary_stats_entries((DICTIONARY *)host->rrdctx_post_processing_queue); + pp_queued_contexts_for_all_hosts += + dictionary_entries((DICTIONARY *)host->rrdctx_post_processing_queue); rrdcontext_post_process_queued_contexts(host); } if(host->rrdctx_hub_queue) { - hub_queued_contexts_for_all_hosts += dictionary_stats_entries((DICTIONARY *)host->rrdctx_hub_queue); + hub_queued_contexts_for_all_hosts += dictionary_entries((DICTIONARY *)host->rrdctx_hub_queue); rrdcontext_dispatch_queued_contexts_to_hub(host, now_ut); } } diff --git a/database/rrdcontext.h b/database/rrdcontext.h index a9e0bd2e36..a9b68a491c 100644 --- a/database/rrdcontext.h +++ b/database/rrdcontext.h @@ -34,6 +34,8 @@ extern void rrdhost_destroy_rrdcontexts(RRDHOST *host); extern void rrdcontext_host_child_connected(RRDHOST *host); extern void rrdcontext_host_child_disconnected(RRDHOST *host); +extern int rrdcontext_foreach_instance_with_rrdset_in_context(RRDHOST *host, const char *context, int (*callback)(RRDSET *st, void *data), void *data); + typedef enum { RRDCONTEXT_OPTION_NONE = 0, RRDCONTEXT_OPTION_SHOW_METRICS = (1 << 0), diff --git a/database/rrddim.c b/database/rrddim.c index b1ebeed49a..1ee9a88b87 100644 --- a/database/rrddim.c +++ b/database/rrddim.c @@ -10,23 +10,303 @@ // ---------------------------------------------------------------------------- // RRDDIM index -static inline void rrddim_index_add(RRDSET *st, RRDDIM *rd) { - if(likely(dictionary_set(st->rrddim_root_index, string2str(rd->id), rd, sizeof(RRDDIM)) == rd)) { - rrddim_flag_set(rd, RRDDIM_FLAG_INDEXED_ID); - } - else { - rrddim_flag_clear(rd, RRDDIM_FLAG_INDEXED_ID); - error("RRDDIM: %s() attempted to index duplicate dimension with key '%s' of chart '%s' of host '%s'", __FUNCTION__, rrddim_id(rd), rrdset_id(st), rrdhost_hostname(st->rrdhost)); +struct rrddim_constructor { + RRDSET *st; + const char *id; + const char *name; + collected_number multiplier; + collected_number divisor; + RRD_ALGORITHM algorithm; + RRD_MEMORY_MODE memory_mode; + + enum { + RRDDIM_REACT_NONE = 0, + RRDDIM_REACT_NEW = (1 << 0), + RRDDIM_REACT_UPDATED = (1 << 2), + } react_action; + +}; + +static void rrddim_update_rrddimvars_unsafe(RRDDIM *rd) { + RRDSET *st = rd->rrdset; + RRDHOST *host = st->rrdhost; + + if(host->health_enabled && !rrdset_is_ar_chart(st)) { + rrddimvar_add_and_leave_released(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_FLAG_NONE); + rrddimvar_add_and_leave_released(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_FLAG_NONE); + rrddimvar_add_and_leave_released(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_FLAG_NONE); + rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARMS); + rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS); + rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS); } } -static inline void rrddim_index_del(RRDSET *st, RRDDIM *rd) { - if(rrddim_flag_check(rd, RRDDIM_FLAG_INDEXED_ID)) { - if (likely(dictionary_del(st->rrddim_root_index, string2str(rd->id)) == 0)) - rrddim_flag_clear(rd, RRDDIM_FLAG_INDEXED_ID); - else - error("RRDDIM: %s() attempted to delete non-indexed dimension with key '%s' of chart '%s' of host '%s'", __FUNCTION__, rrddim_id(rd), rrdset_id(st), rrdhost_hostname(st->rrdhost)); +static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *constructor_data) { + struct rrddim_constructor *ctr = constructor_data; + RRDDIM *rd = rrddim; + RRDSET *st = ctr->st; + RRDHOST *host = st->rrdhost; + + rd->flags = RRDDIM_FLAG_NONE; + + rd->id = string_strdupz(ctr->id); + rd->name = (ctr->name && *ctr->name)?rrd_string_strdupz(ctr->name):string_dup(rd->id); + + rd->algorithm = ctr->algorithm; + rd->multiplier = ctr->multiplier; + rd->divisor = ctr->divisor; + if(!rd->divisor) rd->divisor = 1; + + rd->update_every = st->update_every; + + rd->rrdset = st; + + if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)) + rd->collections_counter = 1; + + if(ctr->memory_mode == RRD_MEMORY_MODE_MAP || ctr->memory_mode == RRD_MEMORY_MODE_SAVE) { + if(!rrddim_memory_load_or_create_map_save(st, rd, ctr->memory_mode)) { + info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (ctr->memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st), rrddim_name(rd)); + ctr->memory_mode = RRD_MEMORY_MODE_RAM; + } } + + if(ctr->memory_mode == RRD_MEMORY_MODE_RAM) { + size_t entries = st->entries; + if(!entries) entries = 5; + + rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1); + if(!rd->db) { + info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd)); + ctr->memory_mode = RRD_MEMORY_MODE_ALLOC; + } + else rd->memsize = entries * sizeof(storage_number); + } + + if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) { + size_t entries = st->entries; + if(entries < 5) entries = 5; + + rd->db = callocz(entries, sizeof(storage_number)); + rd->memsize = entries * sizeof(storage_number); + } + + rd->rrd_memory_mode = ctr->memory_mode; + +#ifdef ENABLE_ACLK + rd->aclk_live_status = -1; +#endif + + (void) find_dimension_uuid(st, rd, &(rd->metric_uuid)); + + // initialize the db tiers + { + size_t initialized = 0; + RRD_MEMORY_MODE wanted_mode = ctr->memory_mode; + for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) { + STORAGE_ENGINE *eng = storage_engine_get(wanted_mode); + if(!eng) continue; + + rd->tiers[tier] = callocz(1, sizeof(struct rrddim_tier)); + rd->tiers[tier]->tier_grouping = get_tier_grouping(tier); + rd->tiers[tier]->mode = eng->id; + rd->tiers[tier]->collect_ops = eng->api.collect_ops; + rd->tiers[tier]->query_ops = eng->api.query_ops; + rd->tiers[tier]->db_metric_handle = eng->api.init(rd, host->storage_instance[tier]); + storage_point_unset(rd->tiers[tier]->virtual_point); + initialized++; + + // internal_error(true, "TIER GROUPING of chart '%s', dimension '%s' for tier %d is set to %d", rd->rrdset->name, rd->name, tier, rd->tiers[tier]->tier_grouping); + } + + if(!initialized) + error("Failed to initialize all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); + + if(!rd->tiers[0]) + error("Failed to initialize the first db tier for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); + } + + // initialize data collection for all tiers + { + size_t initialized = 0; + for (int tier = 0; tier < storage_tiers; tier++) { + if (rd->tiers[tier]) { + rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle); + initialized++; + } + } + + if(!initialized) + error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); + } + + if(st->dimensions) { + RRDDIM *td = st->dimensions; + + if(td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor)) { + if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { +#ifdef NETDATA_INTERNAL_CHECKS + info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").", + rrddim_name(rd), + rrdset_name(st), + rrdhost_hostname(host), + rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(td->algorithm), + rd->multiplier, td->multiplier, + rd->divisor, td->divisor + ); +#endif + rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS); + } + } + } + + rrddim_update_rrddimvars_unsafe(rd); + + // let the chart resync + rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); + rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); + + ml_new_dimension(rd); + + ctr->react_action = RRDDIM_REACT_NEW; + + internal_error(false, "RRDDIM: inserted dimension '%s' of chart '%s' of host '%s'", + rrddim_name(rd), rrdset_name(st), rrdhost_hostname(st->rrdhost)); + +} + +static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *rrdset) { + RRDDIM *rd = rrddim; + RRDSET *st = rrdset; (void)st; + + internal_error(false, "RRDDIM: deleting dimension '%s' of chart '%s' of host '%s'", + rrddim_name(rd), rrdset_name(st), rrdhost_hostname(st->rrdhost)); + + rrdcontext_removed_rrddim(rd); + + ml_delete_dimension(rd); + + debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd)); + + if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { + + size_t tiers_available = 0, tiers_said_yes = 0; + for(int tier = 0; tier < storage_tiers ;tier++) { + if(rd->tiers[tier]) { + tiers_available++; + + if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle)) + tiers_said_yes++; + + rd->tiers[tier]->db_collection_handle = NULL; + } + } + + if (tiers_available == tiers_said_yes && tiers_said_yes && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + /* This metric has no data and no references */ + delete_dimension_uuid(&rd->metric_uuid); + } + } + + rrddimvar_delete_all(rd); + + // free(rd->annotations); + //#ifdef ENABLE_ACLK + // if (!netdata_exit) + // aclk_send_dimension_update(rd); + //#endif + + // this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures + rrddim_memory_file_free(rd); + + for(int tier = 0; tier < storage_tiers ;tier++) { + if(!rd->tiers[tier]) continue; + + STORAGE_ENGINE* eng = storage_engine_get(rd->tiers[tier]->mode); + if(eng) + eng->api.free(rd->tiers[tier]->db_metric_handle); + + freez(rd->tiers[tier]); + rd->tiers[tier] = NULL; + } + + if(rd->db) { + if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM) + munmap(rd->db, rd->memsize); + else + freez(rd->db); + } + + string_freez(rd->id); + string_freez(rd->name); +} + +static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *new_rrddim, void *constructor_data) { + (void)new_rrddim; // it is NULL + + struct rrddim_constructor *ctr = constructor_data; + RRDDIM *rd = rrddim; + RRDSET *st = ctr->st; + + ctr->react_action = RRDDIM_REACT_NONE; + + int rc = rrddim_reset_name(st, rd, ctr->name); + rc += rrddim_set_algorithm(st, rd, ctr->algorithm); + rc += rrddim_set_multiplier(st, rd, ctr->multiplier); + rc += rrddim_set_divisor(st, rd, ctr->divisor); + + if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { + + for(int tier = 0; tier < storage_tiers ;tier++) { + if (rd->tiers[tier]) + rd->tiers[tier]->db_collection_handle = + rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle); + } + + rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED); + rrddim_update_rrddimvars_unsafe(rd); + } + + if(unlikely(rc)) + ctr->react_action = RRDDIM_REACT_UPDATED; + + return ctr->react_action == RRDDIM_REACT_UPDATED; +} + +static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *constructor_data) { + struct rrddim_constructor *ctr = constructor_data; + RRDDIM *rd = rrddim; + RRDSET *st = ctr->st; + + if(ctr->react_action == RRDDIM_REACT_UPDATED) { + debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rrddim_id(rd)); + (void)sql_store_dimension(&rd->metric_uuid, &rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); +#ifdef ENABLE_ACLK + queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec())); +#endif + + // the chart needs to be updated to the parent + rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); + rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); + } + + rrdcontext_updated_rrddim(rd); +} + +void rrddim_index_init(RRDSET *st) { + if(!st->rrddim_root_index) { + st->rrddim_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL); + dictionary_register_conflict_callback(st->rrddim_root_index, rrddim_conflict_callback, NULL); + dictionary_register_delete_callback(st->rrddim_root_index, rrddim_delete_callback, st); + dictionary_register_react_callback(st->rrddim_root_index, rrddim_react_callback, st); + } +} + +void rrddim_index_destroy(RRDSET *st) { + dictionary_destroy(st->rrddim_root_index); + st->rrddim_root_index = NULL; } static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id) { @@ -55,14 +335,15 @@ RRDDIM *rrddim_find_active(RRDSET *st, const char *id) { // ---------------------------------------------------------------------------- // RRDDIM rename a dimension -inline int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) { +inline int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name) { if(unlikely(!name || !*name || !strcmp(rrddim_name(rd), name))) return 0; - debug(D_RRD_CALLS, "rrddim_set_name() from %s.%s to %s.%s", rrdset_name(st), rrddim_name(rd), rrdset_name(st), name); + debug(D_RRD_CALLS, "rrddim_reset_name() from %s.%s to %s.%s", rrdset_name(st), rrddim_name(rd), rrdset_name(st), name); - string_freez(rd->name); + STRING *old = rd->name; rd->name = rrd_string_strdupz(name); + string_freez(old); if (!rrdset_is_ar_chart(st)) rrddimvar_rename_all(rd); @@ -115,34 +396,34 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor) } // ---------------------------------------------------------------------------- -// RRDDIM create a dimension -void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) { - RRDCALC *rc; - - for (rc = host->alarms_with_foreach; rc; rc = rc->next) { - if (simple_pattern_matches(rc->spdim, rrddim_id(rd)) || simple_pattern_matches(rc->spdim, rrddim_name(rd))) { - if (rc->chart == st->name || rc->chart == st->id) { - char *name = alarm_name_with_dim(rrdcalc_name(rc), string_strlen(rc->name), rrddim_name(rd), string_strlen(rd->name)); - if(rrdcalc_exists(host, rrdset_name(st), name)) { - freez(name); - continue; - } +// get the timestamp of the last entry in the round-robin database +time_t rrddim_last_entry_t(RRDDIM *rd) { + time_t latest = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle); - netdata_rwlock_wrlock(&host->health_log.alarm_log_rwlock); - RRDCALC *child = rrdcalc_create_from_rrdcalc(rc, host, name, rrddim_name(rd)); - netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); + for(int tier = 1; tier < storage_tiers ;tier++) { + if(unlikely(!rd->tiers[tier])) continue; - if (child) - rrdcalc_add_to_host(host, child); - - else { - error("Cannot allocate a new alarm."); - rc->foreachcounter--; - } - } - } + time_t t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle); + if(t > latest) + latest = t; } + + return latest; +} + +time_t rrddim_first_entry_t(RRDDIM *rd) { + time_t oldest = 0; + + for(int tier = 0; tier < storage_tiers ;tier++) { + if(unlikely(!rd->tiers[tier])) continue; + + time_t t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle); + if(t != 0 && (oldest == 0 || t < oldest)) + oldest = t; + } + + return oldest; } // Return either @@ -154,12 +435,14 @@ time_t calc_dimension_liveness(RRDDIM *rd, time_t now) { time_t last_updated = rd->last_collected_time.tv_sec; int live; - if (rd->aclk_live_status == 1) - live = - ((now - last_updated) < - MIN(rrdset_free_obsolete_time, RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER * rd->update_every)); + + if(rd->aclk_live_status == 1) { + int offline_time = RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER * rd->update_every; + live = ((now - last_updated) < MIN(rrdset_free_obsolete_time, offline_time)); + } else live = ((now - last_updated) < RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER * rd->update_every); + return live ? 0 : last_updated; } #endif @@ -172,260 +455,37 @@ RRDDIM *rrddim_add_custom(RRDSET *st , RRD_ALGORITHM algorithm , RRD_MEMORY_MODE memory_mode ) { - RRDHOST *host = st->rrdhost; - rrdset_wrlock(st); + struct rrddim_constructor tmp = { + .st = st, + .id = id, + .name = name, + .multiplier = multiplier, + .divisor = divisor, + .algorithm = algorithm, + .memory_mode = memory_mode, + }; - RRDDIM *rd = rrddim_find(st, id); - if(unlikely(rd)) { - debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", rrdset_id(st), name?name:""); + RRDDIM *rd = dictionary_set_advanced(st->rrddim_root_index, tmp.id, -1, NULL, sizeof(RRDDIM), &tmp); - int rc = rrddim_set_name(st, rd, name); - rc += rrddim_set_algorithm(st, rd, algorithm); - rc += rrddim_set_multiplier(st, rd, multiplier); - rc += rrddim_set_divisor(st, rd, divisor); - - if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { - store_active_dimension(&rd->metric_uuid); - - for(int tier = 0; tier < storage_tiers ;tier++) { - if (rd->tiers[tier]) - rd->tiers[tier]->db_collection_handle = - rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle); - } - - rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED); - rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT); - rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT); - rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); - - rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM); - rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS); - rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS); - } - - if (unlikely(rc)) { - debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rrddim_id(rd)); - (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, - rd->algorithm); -#ifdef ENABLE_ACLK - queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec())); -#endif - rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); - } + if(tmp.react_action == RRDDIM_REACT_NEW) { + // append this dimension + rrdset_wrlock(st); + DOUBLE_LINKED_LIST_APPEND_UNSAFE(st->dimensions, rd, prev, next); rrdset_unlock(st); - rrdcontext_updated_rrddim(rd); - return rd; } - rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); - - rd = callocz(1, sizeof(RRDDIM)); - rd->id = string_strdupz(id); - - rd->name = (name && *name)?rrd_string_strdupz(name):string_dup(rd->id); - - rd->algorithm = algorithm; - rd->multiplier = multiplier; - rd->divisor = divisor; - if(!rd->divisor) rd->divisor = 1; - - rd->entries = st->entries; - rd->update_every = st->update_every; - - if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)) - rd->collections_counter = 1; - - rd->rrdset = st; - - if(memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_SAVE) { - if(!rrddim_memory_load_or_create_map_save(st, rd, memory_mode)) { - info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st), rrddim_name(rd)); - memory_mode = RRD_MEMORY_MODE_RAM; - } - } - - if(memory_mode == RRD_MEMORY_MODE_RAM) { - size_t entries = st->entries; - if(!entries) entries = 5; - - rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1); - if(!rd->db) { - info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd)); - memory_mode = RRD_MEMORY_MODE_ALLOC; - } - else rd->memsize = entries * sizeof(storage_number); - } - - if(memory_mode == RRD_MEMORY_MODE_ALLOC || memory_mode == RRD_MEMORY_MODE_NONE) { - size_t entries = st->entries; - if(entries < 5) entries = 5; - - rd->db = callocz(entries, sizeof(storage_number)); - rd->memsize = entries * sizeof(storage_number); - } - - rd->rrd_memory_mode = memory_mode; - -#ifdef ENABLE_ACLK - rd->aclk_live_status = -1; -#endif - - (void) find_dimension_uuid(st, rd, &(rd->metric_uuid)); - - // initialize the db tiers - { - size_t initialized = 0; - RRD_MEMORY_MODE wanted_mode = memory_mode; - for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) { - STORAGE_ENGINE *eng = storage_engine_get(wanted_mode); - if(!eng) continue; - - rd->tiers[tier] = callocz(1, sizeof(struct rrddim_tier)); - rd->tiers[tier]->tier_grouping = get_tier_grouping(tier); - rd->tiers[tier]->mode = eng->id; - rd->tiers[tier]->collect_ops = eng->api.collect_ops; - rd->tiers[tier]->query_ops = eng->api.query_ops; - rd->tiers[tier]->db_metric_handle = eng->api.init(rd, host->storage_instance[tier]); - storage_point_unset(rd->tiers[tier]->virtual_point); - initialized++; - - // internal_error(true, "TIER GROUPING of chart '%s', dimension '%s' for tier %d is set to %d", rd->rrdset->name, rd->name, tier, rd->tiers[tier]->tier_grouping); - } - - if(!initialized) - error("Failed to initialize all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); - - if(!rd->tiers[0]) - error("Failed to initialize the first db tier for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); - } - - store_active_dimension(&rd->metric_uuid); - - // initialize data collection for all tiers - { - size_t initialized = 0; - for (int tier = 0; tier < storage_tiers; tier++) { - if (rd->tiers[tier]) { - rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle); - initialized++; - } - } - - if(!initialized) - error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); - } - - if(st->dimensions) { - RRDDIM *td = st->dimensions; - - if(td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor)) { - if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { - #ifdef NETDATA_INTERNAL_CHECKS - info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").", - rrddim_name(rd), - rrdset_name(st), - rrdhost_hostname(host), - rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(td->algorithm), - rd->multiplier, td->multiplier, - rd->divisor, td->divisor - ); - #endif - rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS); - } - } - } - - // append this dimension - DOUBLE_LINKED_LIST_APPEND_UNSAFE(st->dimensions, rd, prev, next); - - if(host->health_enabled && !rrdset_is_ar_chart(st)) { - rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT); - rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT); - rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); - } - - rrddim_index_add(st, rd); - - rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM); - rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS); - rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS); - - ml_new_dimension(rd); - - rrdset_unlock(st); - rrdcontext_updated_rrddim(rd); return(rd); } // ---------------------------------------------------------------------------- // RRDDIM remove / free a dimension -void rrddim_free(RRDSET *st, RRDDIM *rd) -{ - rrdcontext_removed_rrddim(rd); - ml_delete_dimension(rd); - - debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd)); - - if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { - - size_t tiers_available = 0, tiers_said_yes = 0; - for(int tier = 0; tier < storage_tiers ;tier++) { - if(rd->tiers[tier]) { - tiers_available++; - - if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle)) - tiers_said_yes++; - - rd->tiers[tier]->db_collection_handle = NULL; - } - } - - if (tiers_available == tiers_said_yes && tiers_said_yes && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - /* This metric has no data and no references */ - delete_dimension_uuid(&rd->metric_uuid); - } - } - +void rrddim_free(RRDSET *st, RRDDIM *rd) { + rrdset_wrlock(st); DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->dimensions, rd, prev, next); + rrdset_unlock(st); - while(rd->variables) - rrddimvar_free(rd->variables); - - rrddim_index_del(st, rd); - - // free(rd->annotations); -//#ifdef ENABLE_ACLK -// if (!netdata_exit) -// aclk_send_dimension_update(rd); -//#endif - - // this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures - rrddim_memory_file_free(rd); - - for(int tier = 0; tier < storage_tiers ;tier++) { - if(!rd->tiers[tier]) continue; - - STORAGE_ENGINE* eng = storage_engine_get(rd->tiers[tier]->mode); - if(eng) - eng->api.free(rd->tiers[tier]->db_metric_handle); - - freez(rd->tiers[tier]); - rd->tiers[tier] = NULL; - } - - if(rd->db) { - if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM) - munmap(rd->db, rd->memsize); - else - freez(rd->db); - } - - string_freez(rd->id); - string_freez(rd->name); - freez(rd); + dictionary_del(st->rrddim_root_index, string2str(rd->id)); } @@ -445,7 +505,7 @@ int rrddim_hide(RRDSET *st, const char *id) { if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) (void)sql_set_dimension_option(&rd->metric_uuid, "hidden"); - rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN); + rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN); rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN); rrdcontext_updated_rrddim_flags(rd); return 0; @@ -463,7 +523,7 @@ int rrddim_unhide(RRDSET *st, const char *id) { if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) (void)sql_set_dimension_option(&rd->metric_uuid, NULL); - rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN); + rrddim_option_clear(rd, RRDDIM_OPTION_HIDDEN); rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN); rrdcontext_updated_rrddim_flags(rd); return 0; @@ -478,6 +538,7 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) { } rrddim_flag_set(rd, RRDDIM_FLAG_OBSOLETE); rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS); + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS); rrdcontext_updated_rrddim_flags(rd); } diff --git a/database/rrddimvar.c b/database/rrddimvar.c index 1726612645..449ceeb937 100644 --- a/database/rrddimvar.c +++ b/database/rrddimvar.c @@ -1,84 +1,87 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#define NETDATA_HEALTH_INTERNALS #include "rrd.h" +typedef struct rrddimvar { + struct rrddim *rrddim; + + STRING *prefix; + STRING *suffix; + void *value; + + const RRDVAR_ACQUIRED *rrdvar_local_dim_id; + const RRDVAR_ACQUIRED *rrdvar_local_dim_name; + + const RRDVAR_ACQUIRED *rrdvar_family_id; + const RRDVAR_ACQUIRED *rrdvar_family_name; + const RRDVAR_ACQUIRED *rrdvar_family_context_dim_id; + const RRDVAR_ACQUIRED *rrdvar_family_context_dim_name; + + const RRDVAR_ACQUIRED *rrdvar_host_chart_id_dim_id; + const RRDVAR_ACQUIRED *rrdvar_host_chart_id_dim_name; + const RRDVAR_ACQUIRED *rrdvar_host_chart_name_dim_id; + const RRDVAR_ACQUIRED *rrdvar_host_chart_name_dim_name; + + RRDVAR_FLAGS flags:24; + RRDVAR_TYPE type:8; +} RRDDIMVAR; + // ---------------------------------------------------------------------------- // RRDDIMVAR management // DIMENSION VARIABLES #define RRDDIMVAR_ID_MAX 1024 -static inline void rrddimvar_free_variables(RRDDIMVAR *rs) { +static inline void rrddimvar_free_variables_unsafe(RRDDIMVAR *rs) { RRDDIM *rd = rs->rrddim; RRDSET *st = rd->rrdset; RRDHOST *host = st->rrdhost; // CHART VARIABLES FOR THIS DIMENSION - rrdvar_free(host, st->rrdvar_root_index, rs->var_local_id); - rs->var_local_id = NULL; + if(st->rrdvars) { + rrdvar_release_and_del(st->rrdvars, rs->rrdvar_local_dim_id); + rs->rrdvar_local_dim_id = NULL; - rrdvar_free(host, st->rrdvar_root_index, rs->var_local_name); - rs->var_local_name = NULL; + rrdvar_release_and_del(st->rrdvars, rs->rrdvar_local_dim_name); + rs->rrdvar_local_dim_name = NULL; + } // FAMILY VARIABLES FOR THIS DIMENSION - rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_id); - rs->var_family_id = NULL; + if(st->rrdfamily) { + rrdvar_release_and_del(rrdfamily_rrdvars_dict(st->rrdfamily), rs->rrdvar_family_id); + rs->rrdvar_family_id = NULL; - rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_name); - rs->var_family_name = NULL; + rrdvar_release_and_del(rrdfamily_rrdvars_dict(st->rrdfamily), rs->rrdvar_family_name); + rs->rrdvar_family_name = NULL; - rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_contextid); - rs->var_family_contextid = NULL; + rrdvar_release_and_del(rrdfamily_rrdvars_dict(st->rrdfamily), rs->rrdvar_family_context_dim_id); + rs->rrdvar_family_context_dim_id = NULL; - rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_contextname); - rs->var_family_contextname = NULL; + rrdvar_release_and_del(rrdfamily_rrdvars_dict(st->rrdfamily), rs->rrdvar_family_context_dim_name); + rs->rrdvar_family_context_dim_name = NULL; + } // HOST VARIABLES FOR THIS DIMENSION - rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartidid); - rs->var_host_chartidid = NULL; + if(host->rrdvars && host->health_enabled) { + rrdvar_release_and_del(host->rrdvars, rs->rrdvar_host_chart_id_dim_id); + rs->rrdvar_host_chart_id_dim_id = NULL; - rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartidname); - rs->var_host_chartidname = NULL; + rrdvar_release_and_del(host->rrdvars, rs->rrdvar_host_chart_id_dim_name); + rs->rrdvar_host_chart_id_dim_name = NULL; - rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartnameid); - rs->var_host_chartnameid = NULL; + rrdvar_release_and_del(host->rrdvars, rs->rrdvar_host_chart_name_dim_id); + rs->rrdvar_host_chart_name_dim_id = NULL; - rrdvar_free(host, host->rrdvar_root_index, rs->var_host_chartnamename); - rs->var_host_chartnamename = NULL; - - // KEYS - - string_freez(rs->key_id); - rs->key_id = NULL; - - string_freez(rs->key_name); - rs->key_name = NULL; - - string_freez(rs->key_fullidid); - rs->key_fullidid = NULL; - - string_freez(rs->key_fullidname); - rs->key_fullidname = NULL; - - string_freez(rs->key_contextid); - rs->key_contextid = NULL; - - string_freez(rs->key_contextname); - rs->key_contextname = NULL; - - string_freez(rs->key_fullnameid); - rs->key_fullnameid = NULL; - - string_freez(rs->key_fullnamename); - rs->key_fullnamename = NULL; + rrdvar_release_and_del(host->rrdvars, rs->rrdvar_host_chart_name_dim_name); + rs->rrdvar_host_chart_name_dim_name = NULL; + } } -static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { - rrddimvar_free_variables(rs); +static inline void rrddimvar_update_variables_unsafe(RRDDIMVAR *rs) { + rrddimvar_free_variables_unsafe(rs); RRDDIM *rd = rs->rrddim; RRDSET *st = rd->rrdset; @@ -89,28 +92,28 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // KEYS snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", string2str(rs->prefix), rrddim_id(rd), string2str(rs->suffix)); - rs->key_id = string_strdupz(buffer); + STRING *key_dim_id = string_strdupz(buffer); snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", string2str(rs->prefix), rrddim_name(rd), string2str(rs->suffix)); - rs->key_name = string_strdupz(buffer); + STRING *key_dim_name = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_id(st), string2str(rs->key_id)); - rs->key_fullidid = string_strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_id(st), string2str(key_dim_id)); + STRING *key_chart_id_dim_id = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_id(st), string2str(rs->key_name)); - rs->key_fullidname = string_strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_id(st), string2str(key_dim_name)); + STRING *key_chart_id_dim_name = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_context(st), string2str(rs->key_id)); - rs->key_contextid = string_strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_context(st), string2str(key_dim_id)); + STRING *key_context_dim_id = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_context(st), string2str(rs->key_name)); - rs->key_contextname = string_strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_context(st), string2str(key_dim_name)); + STRING *key_context_dim_name = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_name(st), string2str(rs->key_id)); - rs->key_fullnameid = string_strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_name(st), string2str(key_dim_id)); + STRING *key_chart_name_dim_id = string_strdupz(buffer); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_name(st), string2str(rs->key_name)); - rs->key_fullnamename = string_strdupz(buffer); + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rrdset_name(st), string2str(key_dim_name)); + STRING *key_chart_name_dim_name = string_strdupz(buffer); // CHART VARIABLES FOR THIS DIMENSION // ----------------------------------- @@ -119,8 +122,10 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $id // - $name - rs->var_local_id = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_local_name = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + if(st->rrdvars) { + rs->rrdvar_local_dim_id = rrdvar_add_and_acquire("local", st->rrdvars, key_dim_id, rs->type, RRDVAR_FLAG_NONE, rs->value); + rs->rrdvar_local_dim_name = rrdvar_add_and_acquire("local", st->rrdvars, key_dim_name, rs->type, RRDVAR_FLAG_NONE, rs->value); + } // FAMILY VARIABLES FOR THIS DIMENSION // ----------------------------------- @@ -131,10 +136,12 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $chart-context.id // - $chart-context.name - rs->var_family_id = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_family_name = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_family_contextid = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_family_contextname = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + if(st->rrdfamily) { + rs->rrdvar_family_id = rrdvar_add_and_acquire("family", rrdfamily_rrdvars_dict(st->rrdfamily), key_dim_id, rs->type, RRDVAR_FLAG_NONE, rs->value); + rs->rrdvar_family_name = rrdvar_add_and_acquire("family", rrdfamily_rrdvars_dict(st->rrdfamily), key_dim_name, rs->type, RRDVAR_FLAG_NONE, rs->value); + rs->rrdvar_family_context_dim_id = rrdvar_add_and_acquire("family", rrdfamily_rrdvars_dict(st->rrdfamily), key_context_dim_id, rs->type, RRDVAR_FLAG_NONE, rs->value); + rs->rrdvar_family_context_dim_name = rrdvar_add_and_acquire("family", rrdfamily_rrdvars_dict(st->rrdfamily), key_context_dim_name, rs->type, RRDVAR_FLAG_NONE, rs->value); + } // HOST VARIABLES FOR THIS DIMENSION // ----------------------------------- @@ -145,60 +152,121 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $chart-name.id // - $chart-name.name - rs->var_host_chartidid = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullidid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_host_chartidname = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullidname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_host_chartnameid = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullnameid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); - rs->var_host_chartnamename = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullnamename, rs->type, RRDVAR_OPTION_DEFAULT, rs->value); + if(host->rrdvars && host->health_enabled) { + rs->rrdvar_host_chart_id_dim_id = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_id_dim_id, rs->type, RRDVAR_FLAG_NONE, rs->value); + rs->rrdvar_host_chart_id_dim_name = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_id_dim_name, rs->type, RRDVAR_FLAG_NONE, rs->value); + rs->rrdvar_host_chart_name_dim_id = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_name_dim_id, rs->type, RRDVAR_FLAG_NONE, rs->value); + rs->rrdvar_host_chart_name_dim_name = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_name_dim_name, rs->type, RRDVAR_FLAG_NONE, rs->value); + } + + // free the keys + + string_freez(key_dim_id); + string_freez(key_dim_name); + string_freez(key_chart_id_dim_id); + string_freez(key_chart_id_dim_name); + string_freez(key_context_dim_id); + string_freez(key_context_dim_name); + string_freez(key_chart_name_dim_id); + string_freez(key_chart_name_dim_name); } -RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options) { - RRDSET *st = rd->rrdset; - (void)st; +struct rrddimvar_constructor { + RRDDIM *rrddim; + const char *prefix; + const char *suffix; + void *value; + RRDVAR_FLAGS flags :16; + RRDVAR_TYPE type:8; +}; - debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", rrdset_id(st), rrdset_name(st), rrddim_id(rd), (prefix)?prefix:"", rrddim_name(rd), (suffix)?suffix:""); +static void rrddimvar_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddimvar, void *constructor_data) { + RRDDIMVAR *rs = rrddimvar; + struct rrddimvar_constructor *ctr = constructor_data; + if(!ctr->prefix) ctr->prefix = ""; + if(!ctr->suffix) ctr->suffix = ""; + + rs->prefix = string_strdupz(ctr->prefix); + rs->suffix = string_strdupz(ctr->suffix); + + rs->type = ctr->type; + rs->value = ctr->value; + rs->flags = ctr->flags; + rs->rrddim = ctr->rrddim; + + rrddimvar_update_variables_unsafe(rs); +} + +static bool rrddimvar_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddimvar, void *new_rrddimvar __maybe_unused, void *constructor_data __maybe_unused) { + RRDDIMVAR *rs = rrddimvar; + rrddimvar_update_variables_unsafe(rs); + + return true; +} + +static void rrddimvar_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddimvar, void *rrdset __maybe_unused) { + RRDDIMVAR *rs = rrddimvar; + rrddimvar_free_variables_unsafe(rs); + string_freez(rs->prefix); + string_freez(rs->suffix); +} + +void rrddimvar_index_init(RRDSET *st) { + if(!st->rrddimvar_root_index) { + st->rrddimvar_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(st->rrddimvar_root_index, rrddimvar_insert_callback, NULL); + dictionary_register_conflict_callback(st->rrddimvar_root_index, rrddimvar_conflict_callback, NULL); + dictionary_register_delete_callback(st->rrddimvar_root_index, rrddimvar_delete_callback, st); + } +} + +void rrddimvar_index_destroy(RRDSET *st) { + dictionary_destroy(st->rrddimvar_root_index); + st->rrddimvar_root_index = NULL; +} + +void rrddimvar_add_and_leave_released(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_FLAGS flags) { if(!prefix) prefix = ""; if(!suffix) suffix = ""; - RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR)); + char key[RRDDIMVAR_ID_MAX + 1]; + size_t key_len = snprintfz(key, RRDDIMVAR_ID_MAX, "%s_%s_%s", prefix, rrddim_id(rd), suffix); - rs->prefix = string_strdupz(prefix); - rs->suffix = string_strdupz(suffix); - - rs->type = type; - rs->value = value; - rs->options = options; - rs->rrddim = rd; - - DOUBLE_LINKED_LIST_PREPEND_UNSAFE(rd->variables, rs, prev, next); - - rrddimvar_create_variables(rs); - - return rs; + struct rrddimvar_constructor tmp = { + .suffix = suffix, + .prefix = prefix, + .type = type, + .flags = flags, + .value = value, + .rrddim = rd + }; + dictionary_set_advanced(rd->rrdset->rrddimvar_root_index, key, (ssize_t)(key_len + 1), NULL, sizeof(RRDDIMVAR), &tmp); } void rrddimvar_rename_all(RRDDIM *rd) { RRDSET *st = rd->rrdset; - (void)st; - debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", rrdset_id(st), rrdset_name(st), rrddim_id(rd), rrddim_name(rd)); + debug(D_VARIABLES, "RRDDIMVAR rename for chart id '%s' name '%s', dimension id '%s', name '%s'", rrdset_id(st), rrdset_name(st), rrddim_id(rd), rrddim_name(rd)); - RRDDIMVAR *rs, *next = rd->variables; - while((rs = next)) { - next = rs->next; - rrddimvar_create_variables(rs); + RRDDIMVAR *rs; + dfe_start_write(st->rrddimvar_root_index, rs) { + if(unlikely(rs->rrddim == rd)) + rrddimvar_update_variables_unsafe(rs); } + dfe_done(rs); } -void rrddimvar_free(RRDDIMVAR *rs) { - RRDDIM *rd = rs->rrddim; - debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset), rrddim_id(rd), rrddim_name(rd), string2str(rs->prefix), string2str(rs->suffix)); +void rrddimvar_delete_all(RRDDIM *rd) { + RRDSET *st = rd->rrdset; - rrddimvar_free_variables(rs); + debug(D_VARIABLES, "RRDDIMVAR delete for chart id '%s' name '%s', dimension id '%s', name '%s'", rrdset_id(st), rrdset_name(st), rrddim_id(rd), rrddim_name(rd)); - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(rd->variables, rs, prev, next); - - string_freez(rs->prefix); - string_freez(rs->suffix); - freez(rs); + RRDDIMVAR *rs; + dfe_start_write(st->rrddimvar_root_index, rs) { + if(unlikely(rs->rrddim == rd)) + dictionary_del(st->rrddimvar_root_index, rs_dfe.name); + } + dfe_done(rs); } diff --git a/database/rrddimvar.h b/database/rrddimvar.h index 6af2a14974..ec3592f560 100644 --- a/database/rrddimvar.h +++ b/database/rrddimvar.h @@ -10,48 +10,12 @@ // calculated / processed by the normal data collection process // This means, there will be no speed penalty for using // these variables -struct rrddimvar { - STRING *prefix; - STRING *suffix; - - STRING *key_id; // dimension id - STRING *key_name; // dimension name - STRING *key_contextid; // context + dimension id - STRING *key_contextname; // context + dimension name - STRING *key_fullidid; // chart type.chart id + dimension id - STRING *key_fullidname; // chart type.chart id + dimension name - STRING *key_fullnameid; // chart type.chart name + dimension id - STRING *key_fullnamename; // chart type.chart name + dimension name - - RRDVAR_TYPE type; - void *value; - - RRDVAR_OPTIONS options; - - RRDVAR *var_local_id; - RRDVAR *var_local_name; - - RRDVAR *var_family_id; - RRDVAR *var_family_name; - RRDVAR *var_family_contextid; - RRDVAR *var_family_contextname; - - RRDVAR *var_host_chartidid; - RRDVAR *var_host_chartidname; - RRDVAR *var_host_chartnameid; - RRDVAR *var_host_chartnamename; - - struct rrddim *rrddim; - - struct rrddimvar *next; - struct rrddimvar *prev; -}; - extern void rrddimvar_rename_all(RRDDIM *rd); -extern RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options); -extern void rrddimvar_free(RRDDIMVAR *rs); - +extern void rrddimvar_add_and_leave_released(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_FLAGS flags); +extern void rrddimvar_delete_all(RRDDIM *rd); +extern void rrddimvar_index_init(RRDSET *st); +extern void rrddimvar_index_destroy(RRDSET *st); #endif //NETDATA_RRDDIMVAR_H diff --git a/database/rrdfamily.c b/database/rrdfamily.c index 438f1a07cb..e7d1536c84 100644 --- a/database/rrdfamily.c +++ b/database/rrdfamily.c @@ -3,61 +3,66 @@ #define NETDATA_RRD_INTERNALS #include "rrd.h" +typedef struct rrdfamily { + STRING *family; + DICTIONARY *rrdvars; +} RRDFAMILY; + // ---------------------------------------------------------------------------- // RRDFAMILY index -static inline RRDFAMILY *rrdfamily_index_add(RRDHOST *host, RRDFAMILY *rc) { - return dictionary_set(host->rrdfamily_root_index, string2str(rc->family), rc, sizeof(RRDFAMILY)); +struct rrdfamily_constructor { + const char *family; +}; + +static void rrdfamily_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdfamily, void *constructor_data) { + RRDFAMILY *rf = rrdfamily; + struct rrdfamily_constructor *ctr = constructor_data; + + rf->family = string_strdupz(ctr->family); + rf->rrdvars = rrdvariables_create(); } -static inline RRDFAMILY *rrdfamily_index_del(RRDHOST *host, RRDFAMILY *rc) { - dictionary_del(host->rrdfamily_root_index, string2str(rc->family)); - return rc; +static void rrdfamily_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdfamily, void *rrdhost __maybe_unused) { + RRDFAMILY *rf = rrdfamily; + string_freez(rf->family); + rrdvariables_destroy(rf->rrdvars); + rf->family = NULL; + rf->rrdvars = NULL; } -static inline RRDFAMILY *rrdfamily_index_find(RRDHOST *host, const char *id) { - return dictionary_get(host->rrdfamily_root_index, id); +void rrdfamily_index_init(RRDHOST *host) { + if(!host->rrdfamily_root_index) { + host->rrdfamily_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(host->rrdfamily_root_index, rrdfamily_insert_callback, NULL); + dictionary_register_delete_callback(host->rrdfamily_root_index, rrdfamily_delete_callback, host); + } } +void rrdfamily_index_destroy(RRDHOST *host) { + dictionary_destroy(host->rrdfamily_root_index); + host->rrdfamily_root_index = NULL; +} + + // ---------------------------------------------------------------------------- // RRDFAMILY management -RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id) { - RRDFAMILY *rc = rrdfamily_index_find(host, id); - if(!rc) { - rc = callocz(1, sizeof(RRDFAMILY)); - - rc->family = string_strdupz(id); - - rc->rrdvar_root_index = dictionary_create( - DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE - |DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - ); - - RRDFAMILY *ret = rrdfamily_index_add(host, rc); - if(ret != rc) - error("RRDFAMILY: INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", string2str(rc->family), (ret)?string2str(ret->family):"NONE"); - } - - rc->use_count++; - return rc; +const RRDFAMILY_ACQUIRED *rrdfamily_add_and_acquire(RRDHOST *host, const char *id) { + struct rrdfamily_constructor tmp = { + .family = id, + }; + return (const RRDFAMILY_ACQUIRED *)dictionary_set_and_acquire_item_advanced(host->rrdfamily_root_index, id, -1, NULL, sizeof(RRDFAMILY), &tmp); } -void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc) { - rc->use_count--; - if(!rc->use_count) { - RRDFAMILY *ret = rrdfamily_index_del(host, rc); - if(ret != rc) - error("RRDFAMILY: INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", string2str(rc->family), (ret)?string2str(ret->family):"NONE"); - else { - debug(D_RRD_CALLS, "RRDFAMILY: Cleaning up remaining family variables for host '%s', family '%s'", rrdhost_hostname(host), string2str(rc->family)); - rrdvar_free_remaining_variables(host, rc->rrdvar_root_index); - - dictionary_destroy(rc->rrdvar_root_index); - string_freez(rc->family); - freez(rc); - } - } +void rrdfamily_release(RRDHOST *host, const RRDFAMILY_ACQUIRED *rfa) { + if(unlikely(!rfa)) return; + dictionary_acquired_item_release(host->rrdfamily_root_index, (const DICTIONARY_ITEM *)rfa); } +DICTIONARY *rrdfamily_rrdvars_dict(const RRDFAMILY_ACQUIRED *rfa) { + if(unlikely(!rfa)) return NULL; + RRDFAMILY *rf = dictionary_acquired_item_value((const DICTIONARY_ITEM *)rfa); + return(rf->rrdvars); +} diff --git a/database/rrdhost.c b/database/rrdhost.c index 003b892f6d..529721946c 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -51,18 +51,12 @@ static DICTIONARY *rrdhost_root_index_hostname = NULL; static inline void rrdhost_init() { if(unlikely(!rrdhost_root_index)) { rrdhost_root_index = dictionary_create( - DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE - | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - ); + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); } if(unlikely(!rrdhost_root_index_hostname)) { rrdhost_root_index_hostname = dictionary_create( - DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE - | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - ); + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); } } @@ -70,7 +64,7 @@ static inline void rrdhost_init() { // RRDHOST index by UUID inline long rrdhost_hosts_available(void) { - return dictionary_stats_entries(rrdhost_root_index); + return dictionary_entries(rrdhost_root_index); } inline RRDHOST *rrdhost_find_by_guid(const char *guid) { @@ -91,7 +85,7 @@ static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) { static void rrdhost_index_del_by_guid(RRDHOST *host) { if(rrdhost_flag_check(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID)) { - if(dictionary_del(rrdhost_root_index, host->machine_guid) != 0) + if(!dictionary_del(rrdhost_root_index, host->machine_guid)) error("RRDHOST: %s() failed to delete machine guid '%s' from index", __FUNCTION__, host->machine_guid); rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_MACHINE_GUID); @@ -126,7 +120,7 @@ static inline void rrdhost_index_del_hostname(RRDHOST *host) { if(unlikely(!host->hostname)) return; if(rrdhost_flag_check(host, RRDHOST_FLAG_INDEXED_HOSTNAME)) { - if(dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host)) != 0) + if(!dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host))) error("RRDHOST: %s() failed to delete hostname '%s' from index", __FUNCTION__, rrdhost_hostname(host)); rrdhost_flag_clear(host, RRDHOST_FLAG_INDEXED_HOSTNAME); @@ -206,6 +200,114 @@ void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory // ---------------------------------------------------------------------------- // RRDHOST - add a host +static void rrdhost_initialize_rrdpush(RRDHOST *host, + unsigned int rrdpush_enabled, + char *rrdpush_destination, + char *rrdpush_api_key, + char *rrdpush_send_charts_matching +) { + if(rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_RRDPUSH)) return; + rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_RRDPUSH); + + sender_init(host); + netdata_mutex_init(&host->receiver_lock); + + host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key) ? 1 : 0; + host->rrdpush_send_destination = (host->rrdpush_send_enabled)?strdupz(rrdpush_destination):NULL; + + if (host->rrdpush_send_destination) + host->destinations = destinations_init(host->rrdpush_send_destination); + + host->rrdpush_send_api_key = (host->rrdpush_send_enabled)?strdupz(rrdpush_api_key):NULL; + host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL, SIMPLE_PATTERN_EXACT); + + host->rrdpush_sender_pipe[0] = -1; + host->rrdpush_sender_pipe[1] = -1; + host->rrdpush_sender_socket = -1; + + //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused? +#ifdef ENABLE_HTTPS + host->ssl.conn = NULL; + host->ssl.flags = NETDATA_SSL_START; + host->stream_ssl.conn = NULL; + host->stream_ssl.flags = NETDATA_SSL_START; +#endif +} + +static void rrdhost_initialize_health(RRDHOST *host, + int is_localhost + ) { + if(!host->health_enabled || rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)) return; + rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH); + + rrdfamily_index_init(host); + rrdcalctemplate_index_init(host); + rrdcalc_rrdhost_index_init(host); + + host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never"); + host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never"); + + host->health_log.next_log_id = 1; + host->health_log.next_alarm_id = 1; + host->health_log.max = 1000; + host->health_log.next_log_id = (uint32_t)now_realtime_sec(); + host->health_log.next_alarm_id = 0; + + long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max); + if(n < 10) { + error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", rrdhost_hostname(host), n, host->health_log.max); + config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max); + } + else + host->health_log.max = (unsigned int)n; + + netdata_rwlock_init(&host->health_log.alarm_log_rwlock); + + char filename[FILENAME_MAX + 1]; + + if(!is_localhost) { + int r = mkdir(host->varlib_dir, 0775); + if (r != 0 && errno != EEXIST) + error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir); + } + + { + snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir); + int r = mkdir(filename, 0775); + if(r != 0 && errno != EEXIST) + error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename); + } + + snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir); + host->health_log_filename = strdupz(filename); + + snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir); + host->health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename)); + host->health_default_recipient = string_strdupz("root"); + + // ------------------------------------------------------------------------ + // load health configuration + + health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL); + + if (!file_is_migrated(host->health_log_filename)) { + int rc = sql_create_health_log_table(host); + if (unlikely(rc)) { + error_report("Failed to create health log table in the database"); + health_alarm_log_load(host); + health_alarm_log_open(host); + } + else { + health_alarm_log_load(host); + add_migrated_file(host->health_log_filename, 0); + } + } else { + sql_create_health_log_table(host); + sql_health_alarm_log_load(host); + } +} + + RRDHOST *rrdhost_create(const char *hostname, const char *registry_hostname, const char *guid, @@ -251,40 +353,18 @@ RRDHOST *rrdhost_create(const char *hostname, host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries); host->health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled; - sender_init(host); - netdata_mutex_init(&host->receiver_lock); - - host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key) ? 1 : 0; - host->rrdpush_send_destination = (host->rrdpush_send_enabled)?strdupz(rrdpush_destination):NULL; - if (host->rrdpush_send_destination) - host->destinations = destinations_init(host->rrdpush_send_destination); - host->rrdpush_send_api_key = (host->rrdpush_send_enabled)?strdupz(rrdpush_api_key):NULL; - host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL, SIMPLE_PATTERN_EXACT); - - host->rrdpush_sender_pipe[0] = -1; - host->rrdpush_sender_pipe[1] = -1; - host->rrdpush_sender_socket = -1; - - //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused? -#ifdef ENABLE_HTTPS - host->ssl.conn = NULL; - host->ssl.flags = NETDATA_SSL_START; - host->stream_ssl.conn = NULL; - host->stream_ssl.flags = NETDATA_SSL_START; -#endif + if (likely(!archived)) { + host->rrdlabels = rrdlabels_create(); + rrdhost_initialize_rrdpush( + host, rrdpush_enabled, rrdpush_destination, rrdpush_api_key, rrdpush_send_charts_matching); + } netdata_rwlock_init(&host->rrdhost_rwlock); - if (likely(!archived)) - host->rrdlabels = rrdlabels_create(); - netdata_mutex_init(&host->aclk_state_lock); host->system_info = system_info; - host->rrdset_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); - host->rrdset_root_index_name = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); - host->rrdfamily_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); - host->rrdvar_root_index = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + rrdset_index_init(host); if(config_get_boolean(CONFIG_SECTION_DB, "delete obsolete charts files", 1)) rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); @@ -292,41 +372,17 @@ RRDHOST *rrdhost_create(const char *hostname, if(config_get_boolean(CONFIG_SECTION_DB, "delete orphan hosts files", 1) && !is_localhost) rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST); - host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never"); - host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never"); - - // ------------------------------------------------------------------------ - // initialize health variables - - host->health_log.next_log_id = 1; - host->health_log.next_alarm_id = 1; - host->health_log.max = 1000; - host->health_log.next_log_id = (uint32_t)now_realtime_sec(); - host->health_log.next_alarm_id = 0; - - long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max); - if(n < 10) { - error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", rrdhost_hostname(host), n, host->health_log.max); - config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max); - } - else - host->health_log.max = (unsigned int)n; - - netdata_rwlock_init(&host->health_log.alarm_log_rwlock); - char filename[FILENAME_MAX + 1]; - if(is_localhost) { - host->cache_dir = strdupz(netdata_configured_cache_dir); host->varlib_dir = strdupz(netdata_configured_varlib_dir); - } else { // this is not localhost - append our GUID to localhost path if (is_in_multihost) { // don't append to cache dir in multihost host->cache_dir = strdupz(netdata_configured_cache_dir); - } else { + } + else { snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_cache_dir, host->machine_guid); host->cache_dir = strdupz(filename); } @@ -340,38 +396,11 @@ RRDHOST *rrdhost_create(const char *hostname, snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_varlib_dir, host->machine_guid); host->varlib_dir = strdupz(filename); - - if(host->health_enabled) { - int r = mkdir(host->varlib_dir, 0775); - if(r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir); - } - } - if(host->health_enabled) { - snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir); - int r = mkdir(filename, 0775); - if(r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename); - } - - snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir); - host->health_log_filename = strdupz(filename); - - snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir); - host->health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename)); - host->health_default_recipient = string_strdupz("root"); - - - // ------------------------------------------------------------------------ - // load health configuration - - if(host->health_enabled) { - rrdhost_wrlock(host); - health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL); - rrdhost_unlock(host); - } + // this is also needed for custom host variables - not only health + if(!host->rrdvars) + host->rrdvars = rrdvariables_create(); RRDHOST *t = rrdhost_index_add_by_guid(host); if(t != host) { @@ -382,33 +411,20 @@ RRDHOST *rrdhost_create(const char *hostname, if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) { int rc; - if (!archived) { + + if(!archived) { rc = sql_store_host_info(host); if (unlikely(rc)) error_report("Failed to store machine GUID to the database"); } + sql_load_node_id(host); - if (host->health_enabled) { - if (!file_is_migrated(host->health_log_filename)) { - rc = sql_create_health_log_table(host); - if (unlikely(rc)) { - error_report("Failed to create health log table in the database"); - health_alarm_log_load(host); - health_alarm_log_open(host); - } - else { - health_alarm_log_load(host); - add_migrated_file(host->health_log_filename, 0); - } - } else { - sql_create_health_log_table(host); - sql_health_alarm_log_load(host); - } - } } else error_report("Host machine GUID %s is not valid", host->machine_guid); + rrdhost_initialize_health(host, is_localhost); + if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { #ifdef ENABLE_DBENGINE char dbenginepath[FILENAME_MAX + 1]; @@ -530,6 +546,8 @@ RRDHOST *rrdhost_create(const char *hostname, ml_new_host(host); else rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); + + return host; } @@ -605,50 +623,24 @@ void rrdhost_update(RRDHOST *host // update host tags rrdhost_init_tags(host, tags); + if(!host->rrdvars) + host->rrdvars = rrdvariables_create(); + if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) { rrdhost_flag_clear(host, RRDHOST_FLAG_ARCHIVED); - host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key) ? 1 : 0; - host->rrdpush_send_destination = (host->rrdpush_send_enabled)?strdupz(rrdpush_destination):NULL; - if (host->rrdpush_send_destination) - host->destinations = destinations_init(host->rrdpush_send_destination); - host->rrdpush_send_api_key = (host->rrdpush_send_enabled)?strdupz(rrdpush_api_key):NULL; - host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL, SIMPLE_PATTERN_EXACT); + if(!host->rrdlabels) + host->rrdlabels = rrdlabels_create(); - if(host->health_enabled) { - int r; - char filename[FILENAME_MAX + 1]; + rrdhost_initialize_rrdpush(host, + rrdpush_enabled, + rrdpush_destination, + rrdpush_api_key, + rrdpush_send_charts_matching); - if (host != localhost) { - r = mkdir(host->varlib_dir, 0775); - if (r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir); - } - snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir); - r = mkdir(filename, 0775); - if(r != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename); + rrdhost_initialize_health(host, + host == localhost); - rrdhost_wrlock(host); - health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL); - rrdhost_unlock(host); - - if (!file_is_migrated(host->health_log_filename)) { - int rc = sql_create_health_log_table(host); - if (unlikely(rc)) { - error_report("Failed to create health log table in the database"); - - health_alarm_log_load(host); - health_alarm_log_open(host); - } else { - health_alarm_log_load(host); - add_migrated_file(host->health_log_filename, 0); - } - } else { - sql_create_health_log_table(host); - sql_health_alarm_log_load(host); - } - } rrd_hosts_available++; ml_new_host(host); rrdhost_load_rrdcontext_data(host); @@ -762,32 +754,6 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, tim return 0; } -void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host) { - time_t now = now_realtime_sec(); - - RRDHOST *host; - -restart_after_removal: - rrdhost_foreach_write(host) { - if(rrdhost_should_be_removed(host, protected_host, now)) { - info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", rrdhost_hostname(host), host->machine_guid); - - if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) -#ifdef ENABLE_DBENGINE - /* don't delete multi-host DB host files */ - && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0])) -#endif - ) - rrdhost_delete_charts(host); - else - rrdhost_save_charts(host); - - rrdhost_free(host, 0); - goto restart_after_removal; - } - } -} - // ---------------------------------------------------------------------------- // RRDHOST global / startup initialization @@ -1091,12 +1057,21 @@ void rrdhost_free(RRDHOST *host, bool force) { // ------------------------------------------------------------------------ // clean up streaming + stop_streaming_sender(host); if (netdata_exit || force) stop_streaming_receiver(host); + + // ------------------------------------------------------------------------ + // clean up alarms + + rrdcalc_delete_all(host); + + rrdhost_wrlock(host); // lock this RRDHOST + // ------------------------------------------------------------------------ // release its children resources @@ -1109,27 +1084,13 @@ void rrdhost_free(RRDHOST *host, bool force) { } #endif - while(host->rrdset_root) - rrdset_free(host->rrdset_root); + // delete all the RRDSETs of the host + rrdset_index_destroy(host); + rrdcalc_rrdhost_index_destroy(host); + rrdcalctemplate_index_destroy(host); freez(host->exporting_flags); - while(host->host_alarms) - rrdcalc_unlink_and_free(host, host->host_alarms); - - RRDCALC *rc,*nc; - for(rc = host->alarms_with_foreach; rc ; rc = nc) { - nc = rc->next; - rrdcalc_free(rc); - } - host->alarms_with_foreach = NULL; - - while(host->alarms_templates) - rrdcalctemplate_unlink_and_free(host, host->alarms_templates); - - debug(D_RRD_CALLS, "RRDHOST: Cleaning up remaining host variables for host '%s'", rrdhost_hostname(host)); - rrdvar_free_remaining_variables(host, host->rrdvar_root_index); - health_alarm_log_free(host); #ifdef ENABLE_DBENGINE @@ -1208,10 +1169,8 @@ void rrdhost_free(RRDHOST *host, bool force) { netdata_rwlock_destroy(&host->rrdhost_rwlock); freez(host->node_id); - dictionary_destroy(host->rrdset_root_index); - dictionary_destroy(host->rrdset_root_index_name); - dictionary_destroy(host->rrdfamily_root_index); - dictionary_destroy(host->rrdvar_root_index); + rrdfamily_index_destroy(host); + rrdvariables_destroy(host->rrdvars); rrdhost_destroy_rrdcontexts(host); @@ -1249,15 +1208,10 @@ void rrdhost_save_charts(RRDHOST *host) { // we get a write lock // to ensure only one thread is saving the database - rrdhost_wrlock(host); - rrdset_foreach_write(st, host) { - rrdset_rdlock(st); rrdset_save(st); - rrdset_unlock(st); } - - rrdhost_unlock(host); + rrdset_foreach_done(st); } static void rrdhost_load_auto_labels(void) { @@ -1411,17 +1365,12 @@ void rrdhost_delete_charts(RRDHOST *host) { // we get a write lock // to ensure only one thread is saving the database - rrdhost_wrlock(host); - rrdset_foreach_write(st, host) { - rrdset_rdlock(st); - rrdset_delete_files(st); - rrdset_unlock(st); + rrdset_delete_files(st); } + rrdset_foreach_done(st); recursively_delete_dir(host->cache_dir, "left over host"); - - rrdhost_unlock(host); } // ---------------------------------------------------------------------------- @@ -1437,22 +1386,19 @@ void rrdhost_cleanup_charts(RRDHOST *host) { // we get a write lock // to ensure only one thread is saving the database - rrdhost_wrlock(host); - rrdset_foreach_write(st, host) { - rrdset_rdlock(st); if(rrdhost_delete_obsolete_charts && rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)) rrdset_delete_files(st); + else if(rrdhost_delete_obsolete_charts && rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) rrdset_delete_obsolete_dimensions(st); + else rrdset_save(st); - rrdset_unlock(st); } - - rrdhost_unlock(host); + rrdset_foreach_done(st); } @@ -1496,157 +1442,6 @@ void rrdhost_cleanup_all(void) { } -// ---------------------------------------------------------------------------- -// RRDHOST - save or delete all the host charts from disk - -void rrdhost_cleanup_obsolete_charts(RRDHOST *host) { - time_t now = now_realtime_sec(); - - RRDSET *st; - - uint32_t rrdhost_delete_obsolete_charts = rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); - -restart_after_removal: - rrdset_foreach_write(st, host) { - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) - && st->last_accessed_time + rrdset_free_obsolete_time < now - && st->last_updated.tv_sec + rrdset_free_obsolete_time < now - && st->last_collected_time.tv_sec + rrdset_free_obsolete_time < now - )) { - st->rrdhost->obsolete_charts_count--; -#ifdef ENABLE_DBENGINE - if(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - RRDDIM *rd, *last; - - rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED); - while (st->variables) rrdsetvar_free(st->variables); - while (st->alarms) rrdsetcalc_unlink(st->alarms); - rrdset_wrlock(st); - for (rd = st->dimensions, last = NULL ; likely(rd) ; ) { - if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { - last = rd; - rd = rd->next; - continue; - } - - if (rrddim_flag_check(rd, RRDDIM_FLAG_ACLK)) { - last = rd; - rd = rd->next; - continue; - } - rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED); - while (rd->variables) - rrddimvar_free(rd->variables); - - if (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { - rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE); - - /* only a collector can mark a chart as obsolete, so we must remove the reference */ - - size_t tiers_available = 0, tiers_said_yes = 0; - for(int tier = 0; tier < storage_tiers ;tier++) { - if(rd->tiers[tier]) { - tiers_available++; - - if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle)) - tiers_said_yes++; - - rd->tiers[tier]->db_collection_handle = NULL; - } - } - - if (tiers_available == tiers_said_yes && tiers_said_yes) { - /* This metric has no data and no references */ - delete_dimension_uuid(&rd->metric_uuid); - rrddim_free(st, rd); - if (unlikely(!last)) { - rd = st->dimensions; - } - else { - rd = last->next; - } - continue; - } -#ifdef ENABLE_ACLK - else - queue_dimension_to_aclk(rd, rd->last_collected_time.tv_sec); -#endif - } - last = rd; - rd = rd->next; - } - rrdset_unlock(st); - - debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", rrdhost_hostname(host), rrdset_id(st)); - rrdvar_free_remaining_variables(host, st->rrdvar_root_index); - - rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); - - if (st->dimensions) { - /* If the chart still has dimensions don't delete it from the metadata log */ - continue; - } - } -#endif - rrdset_rdlock(st); - - if(rrdhost_delete_obsolete_charts) - rrdset_delete_files(st); - else - rrdset_save(st); - - rrdset_unlock(st); - - rrdset_free(st); - goto restart_after_removal; - } -#ifdef ENABLE_ACLK - else - sql_check_chart_liveness(st); -#endif - } -} - -void rrdset_check_obsoletion(RRDHOST *host) -{ - RRDSET *st; - time_t last_entry_t; - rrdset_foreach_read(st, host) { - last_entry_t = rrdset_last_entry_t(st); - if (last_entry_t && last_entry_t < host->senders_connect_time) { - rrdset_is_obsolete(st); - } - } -} - -void rrd_cleanup_obsolete_charts() -{ - rrd_rdlock(); - - RRDHOST *host; - rrdhost_foreach_read(host) - { - if (host->obsolete_charts_count) { - rrdhost_wrlock(host); - rrdhost_cleanup_obsolete_charts(host); - rrdhost_unlock(host); - } - - if ( host != localhost && - host->trigger_chart_obsoletion_check && - ((host->senders_last_chart_command && - host->senders_last_chart_command + host->health_delay_up_to < now_realtime_sec()) - || (host->senders_connect_time + 300 < now_realtime_sec())) ) { - rrdhost_rdlock(host); - rrdset_check_obsoletion(host); - rrdhost_unlock(host); - host->trigger_chart_obsoletion_check = 0; - } - } - - rrd_unlock(); -} - // ---------------------------------------------------------------------------- // RRDHOST - set system info from environment variables // system_info fields must be heap allocated or NULL @@ -1786,14 +1581,15 @@ int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, ch // Added for gap-filling, if this proves to be a bottleneck in large-scale systems then we will need to cache // the last entry times as the metric updates, but let's see if it is a problem first. time_t rrdhost_last_entry_t(RRDHOST *h) { - rrdhost_rdlock(h); RRDSET *st; time_t result = 0; + rrdset_foreach_read(st, h) { time_t st_last = rrdset_last_entry_t(st); + if (st_last > result) result = st_last; } - rrdhost_unlock(h); + rrdset_foreach_done(st); return result; } diff --git a/database/rrdlabels.c b/database/rrdlabels.c index 6ab1ea66b4..6472743122 100644 --- a/database/rrdlabels.c +++ b/database/rrdlabels.c @@ -478,9 +478,7 @@ typedef struct rrdlabel { RRDLABEL_SRC label_source; } RRDLABEL; -static void rrdlabel_insert_callback(const char *name, void *value, void *data) { - (void)name; - DICTIONARY *dict = (DICTIONARY *)data; (void)dict; +static void rrdlabel_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *dict_ptr __maybe_unused) { RRDLABEL *lb = (RRDLABEL *)value; // label_value is already allocated by the STRING @@ -488,42 +486,43 @@ static void rrdlabel_insert_callback(const char *name, void *value, void *data) lb->label_source &= ~RRDLABEL_FLAG_OLD; } -static void rrdlabel_delete_callback(const char *name, void *value, void *data) { - (void)name; - DICTIONARY *dict = (DICTIONARY *)data; (void)dict; +static void rrdlabel_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *dict_ptr __maybe_unused) { RRDLABEL *lb = (RRDLABEL *)value; string_freez(lb->label_value); lb->label_value = NULL; } -static void rrdlabel_conflict_callback(const char *name, void *oldvalue, void *newvalue, void *data) { - (void)name; - DICTIONARY *dict = (DICTIONARY *)data; (void)dict; +static bool rrdlabel_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *oldvalue, void *newvalue, void *dict_ptr __maybe_unused) { RRDLABEL *lbold = (RRDLABEL *)oldvalue; RRDLABEL *lbnew = (RRDLABEL *)newvalue; if(lbold->label_value == lbnew->label_value) { // they are the same + lbold->label_source |= lbnew->label_source; lbold->label_source |= RRDLABEL_FLAG_OLD; lbold->label_source &= ~RRDLABEL_FLAG_NEW; // free the new one string_freez(lbnew->label_value); + + return false; } - else { - // they are different - string_freez(lbold->label_value); - lbold->label_value = lbnew->label_value; - lbold->label_source = lbnew->label_source; - lbold->label_source |= RRDLABEL_FLAG_NEW; - lbold->label_source &= ~RRDLABEL_FLAG_OLD; - } + + // they are different + + string_freez(lbold->label_value); + lbold->label_value = lbnew->label_value; + lbold->label_source = lbnew->label_source; + lbold->label_source |= RRDLABEL_FLAG_NEW; + lbold->label_source &= ~RRDLABEL_FLAG_OLD; + + return true; } DICTIONARY *rrdlabels_create(void) { - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + DICTIONARY *dict = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); dictionary_register_insert_callback(dict, rrdlabel_insert_callback, dict); dictionary_register_delete_callback(dict, rrdlabel_delete_callback, dict); dictionary_register_conflict_callback(dict, rrdlabel_conflict_callback, dict); @@ -623,7 +622,7 @@ void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls) { // rrdlabels_get_value_to_buffer_or_null() void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null) { - DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key); + const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key); RRDLABEL *lb = dictionary_acquired_item_value(acquired_item); if(lb && lb->label_value) @@ -638,7 +637,7 @@ void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const // rrdlabels_get_value_to_char_or_null() void rrdlabels_get_value_to_char_or_null(DICTIONARY *labels, char **value, const char *key) { - DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key); + const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key); RRDLABEL *lb = dictionary_acquired_item_value(acquired_item); *value = (lb && lb->label_value) ? strdupz(string2str(lb->label_value)) : NULL; @@ -650,10 +649,7 @@ void rrdlabels_get_value_to_char_or_null(DICTIONARY *labels, char **value, const // rrdlabels_unmark_all() // remove labels RRDLABEL_FLAG_OLD and RRDLABEL_FLAG_NEW from all dictionary items -static int remove_flags_old_new(const char *name, void *value, void *data) { - (void)name; - (void)data; - +static int remove_flags_old_new(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { RRDLABEL *lb = (RRDLABEL *)value; if(lb->label_source & RRDLABEL_FLAG_OLD) lb->label_source &= ~RRDLABEL_FLAG_OLD; @@ -671,12 +667,13 @@ void rrdlabels_unmark_all(DICTIONARY *labels) { // rrdlabels_remove_all_unmarked() // remove dictionary items that are neither old, nor new -static int remove_not_old_not_new_callback(const char *name, void *value, void *data) { +static int remove_not_old_not_new_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *name = dictionary_acquired_item_name(item); DICTIONARY *dict = (DICTIONARY *)data; RRDLABEL *lb = (RRDLABEL *)value; if(!(lb->label_source & (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT))) { - dictionary_del_having_write_lock(dict, name); + dictionary_del(dict, name); return 1; } @@ -696,7 +693,8 @@ struct labels_walkthrough { void *data; }; -static int labels_walkthrough_callback(const char *name, void *value, void *data) { +static int labels_walkthrough_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *name = dictionary_acquired_item_name(item); struct labels_walkthrough *d = (struct labels_walkthrough *)data; RRDLABEL *lb = (RRDLABEL *)value; @@ -728,7 +726,8 @@ int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const // rrdlabels_migrate_to_these() // migrate an existing label list to a new list, INPLACE -static int copy_label_to_dictionary_callback(const char *name, void *value, void *data) { +static int copy_label_to_dictionary_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *name = dictionary_acquired_item_name(item); DICTIONARY *dst = (DICTIONARY *)data; RRDLABEL *lb = (RRDLABEL *)value; labels_add_already_sanitized(dst, name, string2str(lb->label_value), lb->label_source); @@ -765,7 +764,8 @@ struct simple_pattern_match_name_value { char equal; }; -static int simple_pattern_match_name_only_callback(const char *name, void *value, void *data) { +static int simple_pattern_match_name_only_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *name = dictionary_acquired_item_name(item); struct simple_pattern_match_name_value *t = (struct simple_pattern_match_name_value *)data; (void)value; @@ -775,7 +775,8 @@ static int simple_pattern_match_name_only_callback(const char *name, void *value return 0; } -static int simple_pattern_match_name_and_value_callback(const char *name, void *value, void *data) { +static int simple_pattern_match_name_and_value_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *name = dictionary_acquired_item_name(item); struct simple_pattern_match_name_value *t = (struct simple_pattern_match_name_value *)data; RRDLABEL *lb = (RRDLABEL *)value; @@ -841,7 +842,9 @@ bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_patte // ---------------------------------------------------------------------------- // Log all labels -static int rrdlabels_log_label_to_buffer_callback(const char *name, void *value, void *data) { +static int rrdlabels_log_label_to_buffer_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *name = dictionary_acquired_item_name(item); + BUFFER *wb = (BUFFER *)data; RRDLABEL *lb = (RRDLABEL *)value; @@ -891,7 +894,8 @@ struct labels_to_buffer { size_t count; }; -static int label_to_buffer_callback(const char *name, void *value, void *data) { +static int label_to_buffer_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *name = dictionary_acquired_item_name(item); struct labels_to_buffer *t = (struct labels_to_buffer *)data; RRDLABEL *lb = (RRDLABEL *)value; @@ -961,17 +965,30 @@ void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) { if (new_rrdlabels) rrdlabels_migrate_to_these(st->rrdlabels, new_rrdlabels); - rrdcalc_update_rrdlabels(st); - - // TODO - we should also cleanup sqlite from old new_rrdlabels that have been removed - BUFFER *sql_buf = buffer_create(1024); - struct label_str tmp = {.sql = sql_buf, .count = 0 }; - uuid_unparse_lower(*st->chart_uuid, tmp.uuid_str); - rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp); - db_execute(buffer_tostring(sql_buf)); - buffer_free(sql_buf); + rrdset_save_rrdlabels_to_sql(st); } +void rrdset_save_rrdlabels_to_sql(RRDSET *st) { + if(!st->rrdlabels) return; + + size_t old_version = st->rrdlabels_last_saved_version; + size_t new_version = dictionary_version(st->rrdlabels); + + if(new_version != old_version) { + // TODO - we should also cleanup sqlite from old new_rrdlabels that have been removed + + BUFFER *sql_buf = buffer_create(1024); + struct label_str tmp = {.sql = sql_buf, .count = 0}; + uuid_unparse_lower(st->chart_uuid, tmp.uuid_str); + rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp); + db_execute(buffer_tostring(sql_buf)); + buffer_free(sql_buf); + + st->rrdlabels_last_saved_version = new_version; + } +} + + // ---------------------------------------------------------------------------- // rrdlabels unit test diff --git a/database/rrdset.c b/database/rrdset.c index 5cd81cd390..f49ac920e0 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -21,58 +21,397 @@ void __rrdset_check_wrlock(RRDSET *st, const char *file, const char *function, c } -// ---------------------------------------------------------------------------- -// RRDSET index - -static inline void rrdset_index_add(RRDHOST *host, RRDSET *st) { - if(likely(dictionary_set(host->rrdset_root_index, rrdset_id(st), st, sizeof(RRDSET)) == st)) { - rrdset_flag_set(st, RRDSET_FLAG_INDEXED_ID); - } - else { - rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID); - error("RRDSET: %s() attempted to index duplicate object with key '%s'", __FUNCTION__, rrdset_id(st)); - } -} - -static inline void rrdset_index_del(RRDHOST *host, RRDSET *st) { - if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID)) { - if(likely(dictionary_del(host->rrdset_root_index, rrdset_id(st)) == 0)) - rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID); - else - error("RRDSET: %s() attempted to delete non-indexed object with key '%s'", __FUNCTION__, rrdset_id(st)); - } -} - -static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) { - return dictionary_get(host->rrdset_root_index, id); -} - // ---------------------------------------------------------------------------- // RRDSET name index +static void rrdset_name_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost __maybe_unused) { + RRDSET *st = rrdset; + rrdset_flag_set(st, RRDSET_FLAG_INDEXED_NAME); +} +static void rrdset_name_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost __maybe_unused) { + RRDSET *st = rrdset; + rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME); +} + static inline void rrdset_index_add_name(RRDHOST *host, RRDSET *st) { - if(likely(dictionary_set(host->rrdset_root_index_name, rrdset_name(st), st, sizeof(RRDSET)) == st)) { - rrdset_flag_set(st, RRDSET_FLAG_INDEXED_NAME); - } - else { - rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME); - error("RRDSET: %s() attempted to index duplicate object with key '%s'", __FUNCTION__, rrdset_name(st)); - } + if(!st->name) return; + dictionary_set(host->rrdset_root_index_name, rrdset_name(st), st, sizeof(RRDSET)); } static inline void rrdset_index_del_name(RRDHOST *host, RRDSET *st) { - if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID)) { - if(likely(dictionary_del(host->rrdset_root_index_name, rrdset_name(st)) != 0)) - rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME); - else - error("RRDSET: %s() attempted to delete non-index object with key '%s'", __FUNCTION__, rrdset_name(st)); - } + if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_NAME)) + dictionary_del(host->rrdset_root_index_name, rrdset_name(st)); } static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) { return dictionary_get(host->rrdset_root_index_name, name); } +// ---------------------------------------------------------------------------- +// RRDSET index + +static inline void rrdset_update_permanent_labels(RRDSET *st) { + if(!st->rrdlabels) return; + + rrdlabels_add(st->rrdlabels, "_collect_plugin", rrdset_plugin_name(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); + rrdlabels_add(st->rrdlabels, "_collect_module", rrdset_module_name(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); +} + +static STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const char *type, const char *current_name, const char *name) { + if(!name || !*name) return NULL; + + char full_name[RRD_ID_LENGTH_MAX + 1]; + char sanitized_name[CONFIG_MAX_VALUE + 1]; + char new_name[CONFIG_MAX_VALUE + 1]; + + snprintfz(full_name, RRD_ID_LENGTH_MAX, "%s.%s", type, name); + rrdset_strncpyz_name(sanitized_name, full_name, CONFIG_MAX_VALUE); + strncpyz(new_name, sanitized_name, CONFIG_MAX_VALUE); + + if(rrdset_index_find_name(host, new_name)) { + debug(D_RRD_CALLS, "RRDSET: chart name '%s' on host '%s' already exists.", new_name, rrdhost_hostname(host)); + if(!strcmp(chart_full_id, full_name) && (!current_name || !*current_name)) { + unsigned i = 1; + + do { + snprintfz(new_name, CONFIG_MAX_VALUE, "%s_%u", sanitized_name, i); + i++; + } while (rrdset_index_find_name(host, new_name)); + + info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, rrdhost_hostname(host)); + } + else + return NULL; + } + + return string_strdupz(new_name); +} + +struct rrdset_constructor { + RRDHOST *host; + const char *type; + const char *id; + const char *name; + const char *family; + const char *context; + const char *title; + const char *units; + const char *plugin; + const char *module; + long priority; + int update_every; + RRDSET_TYPE chart_type; + RRD_MEMORY_MODE memory_mode; + long history_entries; + + enum { + RRDSET_REACT_NONE = 0, + RRDSET_REACT_NEW = (1 << 0), + RRDSET_REACT_CHART_ARCHIVED_TO_LIVE = (1 << 1), + RRDSET_REACT_PLUGIN_UPDATED = (1 << 2), + RRDSET_REACT_MODULE_UPDATED = (1 << 3), + RRDSET_REACT_CHART_ACTIVATED = (1 << 4), + } react_action; +}; + +// the constructor - the dictionary is write locked while this runs +static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data) { + static STRING *anomaly_rates_chart = NULL; + + if(!unlikely(!anomaly_rates_chart)) + anomaly_rates_chart = string_strdupz(ML_ANOMALY_RATES_CHART_ID); + + struct rrdset_constructor *ctr = constructor_data; + RRDHOST *host = ctr->host; + RRDSET *st = rrdset; + + const char *chart_full_id = dictionary_acquired_item_name(item); + + st->id = string_strdupz(chart_full_id); + + st->name = rrdset_fix_name(host, chart_full_id, ctr->type, NULL, ctr->name); + if(!st->name) + st->name = rrdset_fix_name(host, chart_full_id, ctr->type, NULL, ctr->id); + rrdset_index_add_name(host, st); + + st->parts.id = string_strdupz(ctr->id); + st->parts.type = string_strdupz(ctr->type); + st->parts.name = string_strdupz(ctr->name); + + st->family = (ctr->family && *ctr->family) ? rrd_string_strdupz(ctr->family) : rrd_string_strdupz(ctr->type); + st->context = (ctr->context && *ctr->context) ? rrd_string_strdupz(ctr->context) : rrd_string_strdupz(chart_full_id); + + st->units = rrd_string_strdupz(ctr->units); + st->title = rrd_string_strdupz(ctr->title); + st->plugin_name = rrd_string_strdupz(ctr->plugin); + st->module_name = rrd_string_strdupz(ctr->module); + st->priority = ctr->priority; + + st->cache_dir = rrdset_cache_dir(host, chart_full_id); + st->entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5; + st->update_every = ctr->update_every; + st->rrd_memory_mode = ctr->memory_mode; + + st->chart_type = ctr->chart_type; + st->gap_when_lost_iterations_above = (int) (gap_when_lost_iterations_above + 2); + st->rrdhost = host; + + st->flags = RRDSET_FLAG_SYNC_CLOCK | RRDSET_FLAG_INDEXED_ID; + if(unlikely(st->id == anomaly_rates_chart)) + st->flags |= RRDSET_FLAG_ANOMALY_RATE_CHART; + + netdata_rwlock_init(&st->rrdset_rwlock); + netdata_rwlock_init(&st->alerts.rwlock); + + if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) { + if(!rrdset_memory_load_or_create_map_save(st, st->rrd_memory_mode)) { + info("Failed to use db mode %s for chart '%s', falling back to ram mode.", (st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st)); + st->rrd_memory_mode = RRD_MEMORY_MODE_RAM; + } + } + + if (find_chart_uuid(host, string2str(st->parts.type), string2str(st->parts.id), string2str(st->parts.name), &st->chart_uuid)) + uuid_generate(st->chart_uuid); + update_chart_metadata(&st->chart_uuid, st, string2str(st->parts.id), string2str(st->parts.name)); + + rrddim_index_init(st); + + // chart variables - we need this for data collection to work (collector given chart variables) - not only health + rrdsetvar_index_init(st); + + if(host->health_enabled) { + st->green = NAN; + st->red = NAN; + st->rrdfamily = rrdfamily_add_and_acquire(host, rrdset_family(st)); + st->rrdvars = rrdvariables_create(); + rrddimvar_index_init(st); + } + + st->rrdlabels = rrdlabels_create(); + rrdset_update_permanent_labels(st); + + ctr->react_action = RRDSET_REACT_NEW; +} + +// the destructor - the dictionary is write locked while this runs +static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *rrdhost) { + RRDHOST *host = rrdhost; + RRDSET *st = rrdset; + + rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID); + + // remove it from the name index + rrdset_index_del_name(host, st); + + rrdcalc_unlink_all_rrdset_alerts(st); + + // ------------------------------------------------------------------------ + // the order of destruction is important here + + // 1. delete RRDDIMVAR index - this will speed up the destruction of RRDDIMs + // because each dimension loops to find its own variables in this index. + // There are no references to the items on this index from the dimensions. + // To find their own, they have to walk-through the dictionary. + rrddimvar_index_destroy(st); // destroy the rrddimvar index + + // 2. delete RRDSETVAR index + rrdsetvar_index_destroy(st); // destroy the rrdsetvar index + + // 3. delete RRDVAR index after the above, to avoid triggering its garbage collector (they have references on this) + rrdvariables_destroy(st->rrdvars); // free all variables and destroy the rrdvar dictionary + + // 4. delete RRDFAMILY - this has to be last, because RRDDIMVAR and RRDSETVAR need the reference counter + rrdfamily_release(host, st->rrdfamily); // release the acquired rrdfamily -- has to be after all variables + + // 5. delete RRDDIMs, now their variables are not existing, so this is fast + rrddim_index_destroy(st); // free all the dimensions and destroy the dimensions index + + // 6. this has to be after the dimensions are freed, but before labels are freed (contexts need the labels) + rrdcontext_removed_rrdset(st); // let contexts know + + // 7. destroy the chart labels + rrdlabels_destroy(st->rrdlabels); // destroy the labels, after letting the contexts know + + rrdset_memory_file_free(st); // remove files of db mode save and map + + // ------------------------------------------------------------------------ + // free it + + netdata_rwlock_destroy(&st->rrdset_rwlock); + netdata_rwlock_destroy(&st->alerts.rwlock); + + string_freez(st->id); + string_freez(st->name); + string_freez(st->parts.id); + string_freez(st->parts.type); + string_freez(st->parts.name); + string_freez(st->family); + string_freez(st->title); + string_freez(st->units); + string_freez(st->context); + string_freez(st->plugin_name); + string_freez(st->module_name); + + freez(st->exporting_flags); + freez(st->cache_dir); +} + +// the item to be inserted, is already in the dictionary +// this callback deals with the situation, migrating the existing object to the new values +// the dictionary is write locked while this runs +static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *new_rrdset, void *constructor_data) { + (void)new_rrdset; // it is NULL + + struct rrdset_constructor *ctr = constructor_data; + RRDSET *st = rrdset; + + rrdset_isnot_obsolete(st); + + ctr->react_action = RRDSET_REACT_NONE; + + if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) { + rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED); + ctr->react_action |= RRDSET_REACT_CHART_ACTIVATED; + } + + if (rrdset_reset_name(st, (ctr->name && *ctr->name) ? ctr->name : ctr->id) == 2) + ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE; + + if (unlikely(st->priority != ctr->priority)) { + st->priority = ctr->priority; + ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE; + } + if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && st->update_every != ctr->update_every)) { + st->update_every = ctr->update_every; + ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE; + } + + if(ctr->plugin && *ctr->plugin) { + STRING *old_plugin = st->plugin_name; + st->plugin_name = rrd_string_strdupz(ctr->plugin); + if (old_plugin != st->plugin_name) + ctr->react_action |= RRDSET_REACT_PLUGIN_UPDATED; + string_freez(old_plugin); + } + + if(ctr->module && *ctr->module) { + STRING *old_module = st->module_name; + st->module_name = rrd_string_strdupz(ctr->module); + if (old_module != st->module_name) + ctr->react_action |= RRDSET_REACT_MODULE_UPDATED; + string_freez(old_module); + } + + if(ctr->title && *ctr->title) { + STRING *old_title = st->title; + st->title = rrd_string_strdupz(ctr->title); + if(old_title != st->title) + ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE; + string_freez(old_title); + } + + if(ctr->units && *ctr->units) { + STRING *old_units = st->units; + st->units = rrd_string_strdupz(ctr->units); + if(old_units != st->units) + ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE; + string_freez(old_units); + } + + if(ctr->context && *ctr->context) { + STRING *old_context = st->context; + st->context = rrd_string_strdupz(ctr->context); + if(old_context != st->context) + ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE; + string_freez(old_context); + } + + if(st->chart_type != ctr->chart_type) { + st->chart_type = ctr->chart_type; + ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE; + } + + if(ctr->react_action) + rrdset_flag_clear(st, RRDSET_FLAG_ACLK); + + rrdset_update_permanent_labels(st); + + rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); + rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); + + return ctr->react_action != RRDSET_REACT_NONE; +} + +// this is called after all insertions/conflicts, with the dictionary unlocked, with a reference to RRDSET +// so, any actions requiring locks on other objects, should be placed here +static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data) { + struct rrdset_constructor *ctr = constructor_data; + RRDSET *st = rrdset; + RRDHOST *host = st->rrdhost; + + if(host->health_enabled && (ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_CHART_ACTIVATED))) { + rrdsetvar_add_and_leave_released(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, RRDVAR_FLAG_NONE); + rrdsetvar_add_and_leave_released(st, "collected_total_raw", RRDVAR_TYPE_TOTAL, &st->last_collected_total, RRDVAR_FLAG_NONE); + rrdsetvar_add_and_leave_released(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, RRDVAR_FLAG_NONE); + rrdsetvar_add_and_leave_released(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, RRDVAR_FLAG_NONE); + rrdsetvar_add_and_leave_released(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, RRDVAR_FLAG_NONE); + + rrdcalc_link_matching_alerts_to_rrdset(st); + rrdcalctemplate_link_matching_templates_to_rrdset(st); + } + + if(ctr->react_action & (RRDSET_REACT_CHART_ARCHIVED_TO_LIVE | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) { + debug(D_METADATALOG, "CHART [%s] metadata updated", rrdset_id(st)); + if(unlikely(update_chart_metadata(&st->chart_uuid, st, ctr->id, ctr->name))) + error_report("Failed to update chart metadata in the database"); + } + + rrdcontext_updated_rrdset(st); +} + +void rrdset_index_init(RRDHOST *host) { + if(!host->rrdset_root_index) { + host->rrdset_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(host->rrdset_root_index, rrdset_insert_callback, NULL); + dictionary_register_conflict_callback(host->rrdset_root_index, rrdset_conflict_callback, NULL); + dictionary_register_react_callback(host->rrdset_root_index, rrdset_react_callback, NULL); + dictionary_register_delete_callback(host->rrdset_root_index, rrdset_delete_callback, host); + } + + if(!host->rrdset_root_index_name) { + host->rrdset_root_index_name = dictionary_create( + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(host->rrdset_root_index_name, rrdset_name_insert_callback, host); + dictionary_register_delete_callback(host->rrdset_root_index_name, rrdset_name_delete_callback, host); + } +} + +void rrdset_index_destroy(RRDHOST *host) { + // destroy the name index first + dictionary_destroy(host->rrdset_root_index_name); + host->rrdset_root_index_name = NULL; + + // destroy the id index last + dictionary_destroy(host->rrdset_root_index); + host->rrdset_root_index = NULL; +} + +static inline RRDSET *rrdset_index_add(RRDHOST *host, const char *id, struct rrdset_constructor *st_ctr) { + return dictionary_set_advanced(host->rrdset_root_index, id, -1, NULL, sizeof(RRDSET), st_ctr); +} + +static inline void rrdset_index_del(RRDHOST *host, RRDSET *st) { + if(rrdset_flag_check(st, RRDSET_FLAG_INDEXED_ID)) + dictionary_del(host->rrdset_root_index, rrdset_id(st)); +} + +static RRDSET *rrdset_index_find(RRDHOST *host, const char *id) { + // TODO - the name index should have an acquired dictionary item, not just a pointer to RRDSET + return dictionary_get(host->rrdset_root_index, id); +} + // ---------------------------------------------------------------------------- // RRDSET - find charts @@ -118,53 +457,30 @@ char *rrdset_strncpyz_name(char *to, const char *from, size_t length) { return to; } -int rrdset_set_name(RRDSET *st, const char *name) { +int rrdset_reset_name(RRDSET *st, const char *name) { if(unlikely(!strcmp(rrdset_name(st), name))) return 1; RRDHOST *host = st->rrdhost; - debug(D_RRD_CALLS, "rrdset_set_name() old: '%s', new: '%s'", rrdset_name(st), name); + debug(D_RRD_CALLS, "rrdset_reset_name() old: '%s', new: '%s'", rrdset_name(st), name); - char full_name[RRD_ID_LENGTH_MAX + 1]; - char sanitized_name[CONFIG_MAX_VALUE + 1]; - char new_name[CONFIG_MAX_VALUE + 1]; - - snprintfz(full_name, RRD_ID_LENGTH_MAX, "%s.%s", rrdset_type(st), name); - rrdset_strncpyz_name(sanitized_name, full_name, CONFIG_MAX_VALUE); - strncpyz(new_name, sanitized_name, CONFIG_MAX_VALUE); - - if(rrdset_index_find_name(host, new_name)) { - debug(D_RRD_CALLS, "RRDSET: chart name '%s' on host '%s' already exists.", new_name, rrdhost_hostname(host)); - if(!strcmp(rrdset_id(st), full_name) && !st->name) { - unsigned i = 1; - - do { - snprintfz(new_name, CONFIG_MAX_VALUE, "%s_%u", sanitized_name, i); - i++; - } while (rrdset_index_find_name(host, new_name)); - - info("RRDSET: using name '%s' for chart '%s' on host '%s'.", new_name, full_name, rrdhost_hostname(host)); - } else { - return 0; - } - } + STRING *name_string = rrdset_fix_name(host, rrdset_id(st), rrdset_parts_type(st), string2str(st->name), name); + if(!name_string) return 0; if(st->name) { rrdset_index_del_name(host, st); string_freez(st->name); - st->name = string_strdupz(new_name); + st->name = name_string; rrdsetvar_rename_all(st); } - else { - st->name = string_strdupz(new_name); - } + else + st->name = name_string; - rrdset_wrlock(st); RRDDIM *rd; - rrddim_foreach_write(rd, st) + rrddim_foreach_read(rd, st) rrddimvar_rename_all(rd); - rrdset_unlock(st); + rrddim_foreach_done(rd); rrdset_index_add_name(host, st); @@ -178,6 +494,36 @@ int rrdset_set_name(RRDSET *st, const char *name) { return 2; } +// get the timestamp of the last entry in the round-robin database +time_t rrdset_last_entry_t(RRDSET *st) { + RRDDIM *rd; + time_t last_entry_t = 0; + + rrddim_foreach_read(rd, st) { + time_t t = rrddim_last_entry_t(rd); + if(t > last_entry_t) last_entry_t = t; + } + rrddim_foreach_done(rd); + + return last_entry_t; +} + +// get the timestamp of first entry in the round-robin database +time_t rrdset_first_entry_t(RRDSET *st) { + RRDDIM *rd; + time_t first_entry_t = LONG_MAX; + + rrddim_foreach_read(rd, st) { + time_t t = rrddim_first_entry_t(rd); + if(t < first_entry_t) + first_entry_t = t; + } + rrddim_foreach_done(rd); + + if (unlikely(LONG_MAX == first_entry_t)) return 0; + return first_entry_t; +} + inline void rrdset_is_obsolete(RRDSET *st) { if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))) { info("Cannot obsolete already archived chart %s", rrdset_name(st)); @@ -186,8 +532,9 @@ inline void rrdset_is_obsolete(RRDSET *st) { if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) { rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE); + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS); + st->last_accessed_time = now_realtime_sec(); - st->rrdhost->obsolete_charts_count++; rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); @@ -202,7 +549,6 @@ inline void rrdset_isnot_obsolete(RRDSET *st) { if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) { rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); st->last_accessed_time = now_realtime_sec(); - st->rrdhost->obsolete_charts_count--; rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); @@ -220,11 +566,20 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) { rrdset_flag_clear(st, RRDSET_FLAG_HOMOGENEOUS_CHECK); - RRD_ALGORITHM algorithm = st->dimensions->algorithm; - collected_number multiplier = ABS(st->dimensions->multiplier); - collected_number divisor = ABS(st->dimensions->divisor); + bool init = false, is_heterogeneous = false; + RRD_ALGORITHM algorithm; + collected_number multiplier; + collected_number divisor; rrddim_foreach_read(rd, st) { + if(!init) { + algorithm = rd->algorithm; + multiplier = rd->multiplier; + divisor = ABS(rd->divisor); + init = true; + continue; + } + if(algorithm != rd->algorithm || multiplier != ABS(rd->multiplier) || divisor != ABS(rd->divisor)) { if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { #ifdef NETDATA_INTERNAL_CHECKS @@ -239,12 +594,17 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) { #endif rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS); } - return; + + is_heterogeneous = true; + break; } } + rrddim_foreach_done(rd); - rrdset_flag_clear(st, RRDSET_FLAG_HETEROGENEOUS); - rrdcontext_updated_rrdset_flags(st); + if(!is_heterogeneous) { + rrdset_flag_clear(st, RRDSET_FLAG_HETEROGENEOUS); + rrdcontext_updated_rrdset_flags(st); + } } // ---------------------------------------------------------------------------- @@ -275,6 +635,7 @@ void rrdset_reset(RRDSET *st) { } } } + rrddim_foreach_done(rd); } // ---------------------------------------------------------------------------- @@ -326,89 +687,20 @@ static inline void last_updated_time_align(RRDSET *st) { void rrdset_free(RRDSET *st) { if(unlikely(!st)) return; - - RRDHOST *host = st->rrdhost; - - rrdhost_check_wrlock(host); // make sure we have a write lock on the host - rrdset_wrlock(st); // lock this RRDSET - // info("Removing chart '%s' ('%s')", st->id, st->name); - - // ------------------------------------------------------------------------ - // remove it from the indexes - - rrdset_index_del(host, st); - rrdset_index_del_name(host, st); - - // ------------------------------------------------------------------------ - // free its children structures - - freez(st->exporting_flags); - - while(st->variables) rrdsetvar_free(st->variables); -// while(st->alarms) rrdsetcalc_unlink(st->alarms); - /* We must free all connected alarms here in case this has been an ephemeral chart whose alarm was - * created by a template. This leads to an effective memory leak, which cannot be detected since the - * alarms will still be connected to the host, and freed during shutdown. */ - while(st->alarms) rrdcalc_unlink_and_free(st->rrdhost, st->alarms); - while(st->dimensions) rrddim_free(st, st->dimensions); - - rrdfamily_free(host, st->rrdfamily); - - debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", rrdhost_hostname(host), rrdset_id(st)); - rrdvar_free_remaining_variables(host, st->rrdvar_root_index); - - // ------------------------------------------------------------------------ - // unlink it from the host - - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(host->rrdset_root, st, prev, next); - - rrdset_unlock(st); - - // this has to be after the dimensions are freed - rrdcontext_removed_rrdset(st); - - // ------------------------------------------------------------------------ - // free it - - netdata_rwlock_destroy(&st->rrdset_rwlock); - - rrdset_memory_file_free(st); - rrdlabels_destroy(st->rrdlabels); - - // free directly allocated members - - dictionary_destroy(st->rrddim_root_index); - dictionary_destroy(st->rrdvar_root_index); - - string_freez(st->id); - string_freez(st->name); - string_freez(st->type); - string_freez(st->family); - string_freez(st->title); - string_freez(st->units); - string_freez(st->context); - string_freez(st->plugin_name); - string_freez(st->module_name); - - freez(st->cache_dir); - freez(st->chart_uuid); - - freez(st); + rrdset_index_del(st->rrdhost, st); } void rrdset_save(RRDSET *st) { - rrdset_check_rdlock(st); - rrdset_memory_file_save(st); RRDDIM *rd; rrddim_foreach_read(rd, st) rrddim_memory_file_save(rd); + rrddim_foreach_done(rd); } void rrdset_delete_files(RRDSET *st) { RRDDIM *rd; - rrdset_check_rdlock(st); info("Deleting chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st)); @@ -431,6 +723,7 @@ void rrdset_delete_files(RRDSET *st) { if(unlikely(unlink(cache_filename) == -1)) error("Cannot delete dimension file '%s'", cache_filename); } + rrddim_foreach_done(rd); recursively_delete_dir(st->cache_dir, "left-over chart"); } @@ -438,8 +731,6 @@ void rrdset_delete_files(RRDSET *st) { void rrdset_delete_obsolete_dimensions(RRDSET *st) { RRDDIM *rd; - rrdset_check_rdlock(st); - info("Deleting dimensions of chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st)); rrddim_foreach_read(rd, st) { @@ -451,29 +742,12 @@ void rrdset_delete_obsolete_dimensions(RRDSET *st) { error("Cannot delete dimension file '%s'", cache_filename); } } + rrddim_foreach_done(rd); } // ---------------------------------------------------------------------------- // RRDSET - create a chart -static inline RRDSET *rrdset_find_on_create(RRDHOST *host, const char *fullid) { - RRDSET *st = rrdset_find(host, fullid); - if(unlikely(st)) { - rrdset_isnot_obsolete(st); - debug(D_RRD_CALLS, "RRDSET '%s', already exists.", fullid); - return st; - } - - return NULL; -} - -static inline void rrdset_update_permanent_labels(RRDSET *st) { - if(!st->rrdlabels) return; - - rrdlabels_add(st->rrdlabels, "_collect_plugin", rrdset_plugin_name(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); - rrdlabels_add(st->rrdlabels, "_collect_module", rrdset_module_name(st), RRDLABEL_SRC_AUTO| RRDLABEL_FLAG_PERMANENT); -} - RRDSET *rrdset_create_custom( RRDHOST *host , const char *type @@ -491,7 +765,10 @@ RRDSET *rrdset_create_custom( , RRD_MEMORY_MODE memory_mode , long history_entries ) { - if(!type || !type[0]) { + if (host != localhost) + host->senders_last_chart_command = now_realtime_sec(); + + if(!type || !type[0]) fatal("Cannot create rrd stats without a type: id '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'." , (id && *id)?id:"" , (name && *name)?name:"" @@ -502,10 +779,8 @@ RRDSET *rrdset_create_custom( , (plugin && *plugin)?plugin:"" , (module && *module)?module:"" ); - return NULL; - } - if(!id || !id[0]) { + if(!id || !id[0]) fatal("Cannot create rrd stats without an id: type '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'." , type , (name && *name)?name:"" @@ -516,249 +791,40 @@ RRDSET *rrdset_create_custom( , (plugin && *plugin)?plugin:"" , (module && *module)?module:"" ); - return NULL; - } - - if (host != localhost) { - host->senders_last_chart_command = now_realtime_sec(); - } // ------------------------------------------------------------------------ // check if it already exists - char fullid[RRD_ID_LENGTH_MAX + 1]; - snprintfz(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id); - json_fix_string(fullid); - - int changed_from_archived_to_active = 0; - RRDSET *st = rrdset_find_on_create(host, fullid); - if (st) { - int mark_rebuild = 0; - if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) { - rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED); - changed_from_archived_to_active = 1; - mark_rebuild |= META_CHART_ACTIVATED; - } - - int rc; - if(unlikely(name && *name)) - rc = rrdset_set_name(st, name); - else - rc = rrdset_set_name(st, id); - - if (rc == 2) - mark_rebuild |= META_CHART_UPDATED; - - if (unlikely(st->priority != priority)) { - st->priority = priority; - mark_rebuild |= META_CHART_UPDATED; - } - if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && st->update_every != update_every)) { - st->update_every = update_every; - mark_rebuild |= META_CHART_UPDATED; - } - - if(plugin && *plugin) { - STRING *old_plugin = st->plugin_name; - st->plugin_name = rrd_string_strdupz(plugin); - if (old_plugin != st->plugin_name) - mark_rebuild |= META_PLUGIN_UPDATED; - string_freez(old_plugin); - } - - if(module && *module) { - STRING *old_module = st->module_name; - st->module_name = rrd_string_strdupz(module); - if (old_module != st->module_name) - mark_rebuild |= META_MODULE_UPDATED; - string_freez(old_module); - } - - if(title && *title) { - STRING *old_title = st->title; - st->title = rrd_string_strdupz(title); - if(old_title != st->title) - mark_rebuild |= META_CHART_UPDATED; - string_freez(old_title); - } - - if(units && *units) { - STRING *old_units = st->units; - st->units = rrd_string_strdupz(units); - if(old_units != st->units) - mark_rebuild |= META_CHART_UPDATED; - string_freez(old_units); - } - - if(context && *context) { - STRING *old_context = st->context; - st->context = rrd_string_strdupz(context); - if(old_context != st->context) - mark_rebuild |= META_CHART_UPDATED; - string_freez(old_context); - } - - if (st->chart_type != chart_type) { - st->chart_type = chart_type; - mark_rebuild |= META_CHART_UPDATED; - } - - if (mark_rebuild) { - rrdset_flag_clear(st, RRDSET_FLAG_ACLK); - if (mark_rebuild != META_CHART_ACTIVATED) { - info("Collector updated metadata for chart %s", rrdset_id(st)); - sched_yield(); - } - } - if (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED)) { - debug(D_METADATALOG, "CHART [%s] metadata updated", rrdset_id(st)); - rc = update_chart_metadata(st->chart_uuid, st, id, name); - if (unlikely(rc)) - error_report("Failed to update chart metadata in the database"); - - if (!changed_from_archived_to_active) { - rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); - } - } - /* Fall-through during switch from archived to active so that the host lock is taken and health is linked */ - if (!changed_from_archived_to_active) { - rrdset_update_permanent_labels(st); - rrdcontext_updated_rrdset(st); - return st; - } - } - - rrdhost_wrlock(host); - - st = rrdset_find_on_create(host, fullid); - if(st) { - if (changed_from_archived_to_active) { - rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED); - rrdsetvar_create(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); - rrdsetvar_create(st, "collected_total_raw", RRDVAR_TYPE_TOTAL, &st->last_collected_total, RRDVAR_OPTION_DEFAULT); - rrdsetvar_create(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, RRDVAR_OPTION_DEFAULT); - rrdsetvar_create(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, RRDVAR_OPTION_DEFAULT); - rrdsetvar_create(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, RRDVAR_OPTION_DEFAULT); - rrdsetcalc_link_matching(st); - rrdcalctemplate_link_matching(st); - } - rrdhost_unlock(host); - rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); - rrdcontext_updated_rrdset(st); - return st; - } + char full_id[RRD_ID_LENGTH_MAX + 1]; + snprintfz(full_id, RRD_ID_LENGTH_MAX, "%s.%s", type, id); // ------------------------------------------------------------------------ - // get the options from the config, we need to create it - - long entries = 5; - if (memory_mode != RRD_MEMORY_MODE_DBENGINE) - entries = align_entries_to_pagesize(memory_mode, history_entries); - - char *cache_dir = rrdset_cache_dir(host, fullid); - - // ------------------------------------------------------------------------ - // load it or allocate it + // allocate it debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id); - st = callocz(1, sizeof(RRDSET)); + struct rrdset_constructor tmp = { + .host = host, + .type = type, + .id = id, + .name = name, + .family = family, + .context = context, + .title = title, + .units = units, + .plugin = plugin, + .module = module, + .priority = priority, + .update_every = update_every, + .chart_type = chart_type, + .memory_mode = memory_mode, + .history_entries = history_entries, + }; - st->id = string_strdupz(fullid); // fullid is already json_fix'ed - - st->rrdhost = host; - st->cache_dir = cache_dir; - st->entries = entries; - st->update_every = update_every; - - if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP) { - if(!rrdset_memory_load_or_create_map_save(st, memory_mode)) { - info("Failed to use memory mode %s for chart '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st)); - memory_mode = RRD_MEMORY_MODE_RAM; - } - } - st->rrd_memory_mode = memory_mode; - - st->plugin_name = rrd_string_strdupz(plugin); - st->module_name = rrd_string_strdupz(module); - st->chart_type = chart_type; - st->type = rrd_string_strdupz(type); - st->family = family ? rrd_string_strdupz(family) : string_dup(st->type); - - if(strcmp(rrdset_id(st), ML_ANOMALY_RATES_CHART_ID) == 0) - rrdset_flag_set(st, RRDSET_FLAG_ANOMALY_RATE_CHART); - - st->units = rrd_string_strdupz(units); - - st->context = (context && *context) ? rrd_string_strdupz(context) : string_dup(st->id); - - st->priority = priority; - - rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); - - st->green = NAN; - st->red = NAN; - - st->gap_when_lost_iterations_above = (int) (gap_when_lost_iterations_above + 2); - - st->rrddim_root_index = dictionary_create( - DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE - | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - ); - - st->rrdvar_root_index = dictionary_create( - DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE - | DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - ); - - netdata_rwlock_init(&st->rrdset_rwlock); - st->rrdlabels = rrdlabels_create(); - rrdset_update_permanent_labels(st); - - if(name && *name && rrdset_set_name(st, name)) - // we did set the name - ; - else - // could not use the name, use the id - rrdset_set_name(st, id); - - st->title = rrd_string_strdupz(title); - - st->rrdfamily = rrdfamily_create(host, rrdset_family(st)); - - DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->rrdset_root, st, prev, next); - - if(host->health_enabled) { - rrdsetvar_create(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); - rrdsetvar_create(st, "collected_total_raw", RRDVAR_TYPE_TOTAL, &st->last_collected_total, RRDVAR_OPTION_DEFAULT); - rrdsetvar_create(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, RRDVAR_OPTION_DEFAULT); - rrdsetvar_create(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, RRDVAR_OPTION_DEFAULT); - rrdsetvar_create(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, RRDVAR_OPTION_DEFAULT); - } - - rrdset_index_add(host, st); - rrdsetcalc_link_matching(st); - rrdcalctemplate_link_matching(st); - - st->chart_uuid = find_chart_uuid(host, type, id, name); - if (unlikely(!st->chart_uuid)) - st->chart_uuid = create_chart_uuid(st, id, name); - else - update_chart_metadata(st->chart_uuid, st, id, name); - - store_active_chart(st->chart_uuid); - compute_chart_hash(st); - - rrdhost_unlock(host); - rrdcontext_updated_rrdset(st); + RRDSET *st = rrdset_index_add(host, full_id, &tmp); return(st); } - // ---------------------------------------------------------------------------- // RRDSET - data collection iteration control @@ -1030,8 +1096,36 @@ static void store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, rrdcontext_collected_rrddim(rd); } +struct rda_item { + const DICTIONARY_ITEM *item; + RRDDIM *rd; +}; + +static __thread struct rda_item *thread_rda = NULL; +static __thread size_t thread_rda_entries = 0; + +struct rda_item *rrdset_thread_rda(size_t *dimensions) { + + if(unlikely(!thread_rda || (*dimensions) > thread_rda_entries)) { + freez(thread_rda); + thread_rda = mallocz((*dimensions) * sizeof(struct rda_item)); + thread_rda_entries = *dimensions; + } + + *dimensions = thread_rda_entries; + return thread_rda; +} + +void rrdset_thread_rda_free(void) { + freez(thread_rda); + thread_rda = NULL; + thread_rda_entries = 0; +} + static inline size_t rrdset_done_interpolate( RRDSET *st + , struct rda_item *rda_base + , size_t rda_slots , usec_t update_every_ut , usec_t last_stored_ut , usec_t next_store_ut @@ -1068,9 +1162,11 @@ static inline size_t rrdset_done_interpolate( last_ut = next_store_ut; - rrddim_foreach_read(rd, st) { - if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) - continue; + struct rda_item *rda; + size_t dim_id; + for(dim_id = 0, rda = rda_base, rd = rda->rd ; dim_id < rda_slots ; ++dim_id, ++rda) { + rd = rda->rd; + if(unlikely(!rd)) continue; NETDATA_DOUBLE new_value; @@ -1226,6 +1322,7 @@ static inline void rrdset_done_fill_the_gap(RRDSET *st) { #endif } } + rrddim_foreach_done(rd); if(c > 0) { c--; @@ -1258,13 +1355,14 @@ void rrdset_done(RRDSET *st) { netdata_thread_disable_cancelability(); - // a read lock is OK here - rrdset_rdlock(st); - #ifdef ENABLE_ACLK - if (likely(!rrdset_is_ar_chart(st))) { - if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) { - if (likely(st->dimensions && st->counter_done && !queue_chart_to_aclk(st))) { + time_t mark = now_realtime_sec(); + bool rrdset_flag_aclk = rrdset_flag_check(st, RRDSET_FLAG_ACLK); + bool rrdset_is_ar = rrdset_is_ar_chart(st); + + if (likely(!rrdset_is_ar)) { + if (unlikely(!rrdset_flag_aclk)) { + if (likely(rrdset_number_of_dimensions(st) && st->counter_done && !queue_chart_to_aclk(st))) { rrdset_flag_set(st, RRDSET_FLAG_ACLK); } } @@ -1414,6 +1512,42 @@ after_first_database_work: if(unlikely(st->rrdhost->rrdpush_send_enabled)) rrdset_done_push(st); + size_t rda_slots = dictionary_entries(st->rrddim_root_index); + struct rda_item *rda_base = rrdset_thread_rda(&rda_slots); + + size_t dim_id; + size_t dimensions = 0; + struct rda_item *rda = rda_base; + st->collected_total = 0; + rrddim_foreach_read(rd, st) { + if(rd_dfe.counter >= rda_slots) + break; + + rda = &rda_base[dimensions++]; + + if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { + rda->item = NULL; + rda->rd = NULL; + continue; + } + + // store the dimension in the array + rda->item = dictionary_acquired_item_dup(st->rrddim_root_index, rd_dfe.item); + rda->rd = dictionary_acquired_item_value(rda->item); + + // calculate totals + if(likely(rd->updated)) { + st->collected_total += rd->collected_value; + + if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) { + error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st)); + rrddim_isnot_obsolete(st, rd); + } + } + } + rrddim_foreach_done(rd); + rda_slots = dimensions; + if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE)) goto after_second_database_work; @@ -1424,38 +1558,20 @@ after_first_database_work: rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC); #endif - // calculate totals and count the dimensions - int dimensions = 0; - st->collected_total = 0; - rrddim_foreach_read(rd, st) { - if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) - continue; - - dimensions++; - - if(likely(rd->updated)) - st->collected_total += rd->collected_value; - } - uint32_t has_reset_value = 0; // process all dimensions to calculate their values // based on the collected figures only // at this stage we do not interpolate anything - rrddim_foreach_read(rd, st) { - if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) - continue; + for(dim_id = 0, rda = rda_base, rd = rda->rd ; dim_id < rda_slots ; ++dim_id, ++rda) { + rd = rda->rd; + if(unlikely(!rd)) continue; if(unlikely(!rd->updated)) { rd->calculated_value = 0; continue; } - if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) { - error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st)); - rrddim_isnot_obsolete(st, rd); - } - #ifdef NETDATA_INTERNAL_CHECKS rrdset_debug(st, "%s: START " " last_collected_value = " COLLECTED_NUMBER_FORMAT @@ -1530,7 +1646,7 @@ after_first_database_work: , rd->last_collected_value , rd->collected_value); - if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS))) + if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS))) has_reset_value = 1; uint64_t last = (uint64_t)rd->last_collected_value; @@ -1599,7 +1715,7 @@ after_first_database_work: , rd->collected_value ); - if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS))) + if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS))) has_reset_value = 1; rd->last_collected_value = rd->collected_value; @@ -1669,7 +1785,10 @@ after_first_database_work: // } // #endif - rrdset_done_interpolate(st + rrdset_done_interpolate( + st + , rda_base + ,rda_slots , update_every_ut , last_stored_ut , next_store_ut @@ -1682,17 +1801,13 @@ after_first_database_work: after_second_database_work: st->last_collected_total = st->collected_total; -#ifdef ENABLE_ACLK - time_t mark = now_realtime_sec(); -#endif - - rrddim_foreach_read(rd, st) { - if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) - continue; + for(dim_id = 0, rda = rda_base, rd = rda->rd ; dim_id < rda_slots ; ++dim_id, ++rda) { + rd = rda->rd; + if(unlikely(!rd)) continue; #ifdef ENABLE_ACLK - if (likely(!rrdset_is_ar_chart(st))) { - if (!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) && likely(rrdset_flag_check(st, RRDSET_FLAG_ACLK))) + if (likely(!rrdset_is_ar)) { + if (!rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) && likely(rrdset_flag_aclk)) queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, mark)); } #endif @@ -1760,97 +1875,25 @@ after_second_database_work: // update the memory mapped files with the latest values rrdset_memory_file_update(st); - rrddim_foreach_read(rd, st) { + + for(dim_id = 0, rda = rda_base, rd = rda->rd ; dim_id < rda_slots ; ++dim_id, ++rda) { + rd = rda->rd; + if(unlikely(!rd)) continue; rrddim_memory_file_update(rd); } } - // find if there are any obsolete dimensions - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) { - rrddim_foreach_read(rd, st) - if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) - break; + for(dim_id = 0, rda = rda_base, rd = rda->rd ; dim_id < rda_slots ; ++dim_id, ++rda) { + rd = rda->rd; + if(unlikely(!rd)) continue; - if(unlikely(rd)) { - time_t now = now_realtime_sec(); - - RRDDIM *last; - // there is a dimension to free - // upgrade our read lock to a write lock - rrdset_unlock(st); - rrdset_wrlock(st); - - for( rd = st->dimensions, last = NULL ; likely(rd) ; ) { - if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && !rrddim_flag_check(rd, RRDDIM_FLAG_ACLK) - && (rd->last_collected_time.tv_sec + rrdset_free_obsolete_time < now))) { - info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rrddim_name(rd), rrddim_id(rd), rrdset_name(st), rrdset_id(st)); - - const char *cache_filename = rrddim_cache_filename(rd); - if(cache_filename) { - info("Deleting dimension file '%s'.", cache_filename); - if (unlikely(unlink(cache_filename) == -1)) - error("Cannot delete dimension file '%s'", cache_filename); - } - -#ifdef ENABLE_DBENGINE - if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED); - while(rd->variables) - rrddimvar_free(rd->variables); - - rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE); - /* only a collector can mark a chart as obsolete, so we must remove the reference */ - - size_t tiers_available = 0, tiers_said_yes = 0; - for(int tier = 0; tier < storage_tiers ;tier++) { - if(rd->tiers[tier]) { - tiers_available++; - - if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle)) - tiers_said_yes++; - - rd->tiers[tier]->db_collection_handle = NULL; - } - } - - if (tiers_available == tiers_said_yes && tiers_said_yes) { - /* This metric has no data and no references */ - delete_dimension_uuid(&rd->metric_uuid); - } else { - /* Do not delete this dimension */ -#ifdef ENABLE_ACLK - queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, mark)); -#endif - last = rd; - rd = rd->next; - continue; - } - } -#endif - if(unlikely(!last)) { - rrddim_free(st, rd); - rd = st->dimensions; - continue; - } - else { - rrddim_free(st, rd); - rd = last->next; - continue; - } - } - - last = rd; - rd = rd->next; - } - } - else { - rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS); - } + dictionary_acquired_item_release(st->rrddim_root_index, rda->item); + rda->item = NULL; + rda->rd = NULL; } rrdcontext_collected_rrdset(st); - rrdset_unlock(st); netdata_thread_enable_cancelability(); } diff --git a/database/rrdsetvar.c b/database/rrdsetvar.c index 7234cd94ee..b1800fc7a6 100644 --- a/database/rrdsetvar.c +++ b/database/rrdsetvar.c @@ -1,170 +1,268 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#define NETDATA_HEALTH_INTERNALS #include "rrd.h" -// ---------------------------------------------------------------------------- -// RRDSETVAR management -// CHART VARIABLES +typedef struct rrdsetvar { + STRING *name; // variable name + void *value; // we need this to maintain the allocation for custom chart variables -static inline void rrdsetvar_free_variables(RRDSETVAR *rs) { - RRDSET *st = rs->rrdset; + const RRDVAR_ACQUIRED *rrdvar_local; + const RRDVAR_ACQUIRED *rrdvar_family_chart_id; + const RRDVAR_ACQUIRED *rrdvar_family_chart_name; + const RRDVAR_ACQUIRED *rrdvar_host_chart_id; + const RRDVAR_ACQUIRED *rrdvar_host_chart_name; + + RRDVAR_FLAGS flags:24; + RRDVAR_TYPE type:8; +} RRDSETVAR; + +// should only be called while the rrdsetvar dict is write locked +// otherwise, 2+ threads may be setting the same variables at the same time +static inline void rrdsetvar_free_rrdvars_unsafe(RRDSET *st, RRDSETVAR *rs) { RRDHOST *host = st->rrdhost; // ------------------------------------------------------------------------ // CHART - rrdvar_free(host, st->rrdvar_root_index, rs->var_local); - rs->var_local = NULL; + + if(st->rrdvars) { + rrdvar_release_and_del(st->rrdvars, rs->rrdvar_local); + rs->rrdvar_local = NULL; + } // ------------------------------------------------------------------------ // FAMILY - rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family); - rs->var_family = NULL; - rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rs->var_family_name); - rs->var_family_name = NULL; + if(st->rrdfamily) { + rrdvar_release_and_del(rrdfamily_rrdvars_dict(st->rrdfamily), rs->rrdvar_family_chart_id); + rs->rrdvar_family_chart_id = NULL; + + rrdvar_release_and_del(rrdfamily_rrdvars_dict(st->rrdfamily), rs->rrdvar_family_chart_name); + rs->rrdvar_family_chart_name = NULL; + } // ------------------------------------------------------------------------ // HOST - rrdvar_free(host, host->rrdvar_root_index, rs->var_host); - rs->var_host = NULL; - rrdvar_free(host, host->rrdvar_root_index, rs->var_host_name); - rs->var_host_name = NULL; + if(host->rrdvars && host->health_enabled) { + rrdvar_release_and_del(host->rrdvars, rs->rrdvar_host_chart_id); + rs->rrdvar_host_chart_id = NULL; - // ------------------------------------------------------------------------ - // KEYS - string_freez(rs->key_fullid); - rs->key_fullid = NULL; - - string_freez(rs->key_fullname); - rs->key_fullname = NULL; + rrdvar_release_and_del(host->rrdvars, rs->rrdvar_host_chart_name); + rs->rrdvar_host_chart_name = NULL; + } } -static inline void rrdsetvar_create_variables(RRDSETVAR *rs) { - RRDSET *st = rs->rrdset; +// should only be called while the rrdsetvar dict is write locked +// otherwise, 2+ threads may be setting the same variables at the same time +static inline void rrdsetvar_update_rrdvars_unsafe(RRDSET *st, RRDSETVAR *rs) { RRDHOST *host = st->rrdhost; - RRDVAR_OPTIONS options = rs->options; - if(rs->options & RRDVAR_OPTION_ALLOCATED) - options &= ~ RRDVAR_OPTION_ALLOCATED; + RRDVAR_FLAGS options = rs->flags; + options &= ~RRDVAR_OPTIONS_REMOVED_WHEN_PROPAGATING_TO_RRDVAR; // ------------------------------------------------------------------------ // free the old ones (if any) - rrdsetvar_free_variables(rs); + rrdsetvar_free_rrdvars_unsafe(st, rs); // ------------------------------------------------------------------------ // KEYS char buffer[RRDVAR_MAX_LENGTH + 1]; - snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), string2str(rs->variable)); - rs->key_fullid = string_strdupz(buffer); + snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), string2str(rs->name)); + STRING *key_chart_id = string_strdupz(buffer); - snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), string2str(rs->variable)); - rs->key_fullname = string_strdupz(buffer); + snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), string2str(rs->name)); + STRING *key_chart_name = string_strdupz(buffer); // ------------------------------------------------------------------------ // CHART - rs->var_local = rrdvar_create_and_index("local", st->rrdvar_root_index, rs->variable, rs->type, options, rs->value); + + if(st->rrdvars) { + rs->rrdvar_local = rrdvar_add_and_acquire("local", st->rrdvars, rs->name, rs->type, options, rs->value); + } // ------------------------------------------------------------------------ // FAMILY - rs->var_family = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value); - rs->var_family_name = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value); + + if(st->rrdfamily) { + rs->rrdvar_family_chart_id = rrdvar_add_and_acquire("family", rrdfamily_rrdvars_dict(st->rrdfamily), key_chart_id, rs->type, options, rs->value); + rs->rrdvar_family_chart_name = rrdvar_add_and_acquire("family", rrdfamily_rrdvars_dict(st->rrdfamily), key_chart_name, rs->type, options, rs->value); + } // ------------------------------------------------------------------------ // HOST - rs->var_host = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value); - rs->var_host_name = rrdvar_create_and_index("host", host->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value); + + if(host->rrdvars && host->health_enabled) { + rs->rrdvar_host_chart_id = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_id, rs->type, options, rs->value); + rs->rrdvar_host_chart_name = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_name, rs->type, options, rs->value); + } + + // free the keys + string_freez(key_chart_id); + string_freez(key_chart_name); } -RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options) { - debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", rrdset_id(st), rrdset_name(st), variable); - RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR)); +static void rrdsetvar_free_value_unsafe(RRDSETVAR *rs) { + if(rs->flags & RRDVAR_FLAG_ALLOCATED) { + void *old = rs->value; + rs->value = NULL; + rs->flags &= ~RRDVAR_FLAG_ALLOCATED; + freez(old); + } +} - rs->variable = string_strdupz(variable); - rs->type = type; - rs->value = value; - rs->options = options; - rs->rrdset = st; +static void rrdsetvar_set_value_unsafe(RRDSETVAR *rs, void *new_value) { + rrdsetvar_free_value_unsafe(rs); - DOUBLE_LINKED_LIST_PREPEND_UNSAFE(st->variables, rs, prev, next); + if(new_value) + rs->value = new_value; + else { + NETDATA_DOUBLE *n = mallocz(sizeof(NETDATA_DOUBLE)); + *n = NAN; + rs->value = n; + rs->flags |= RRDVAR_FLAG_ALLOCATED; + } +} - rrdsetvar_create_variables(rs); +struct rrdsetvar_constructor { + RRDSET *rrdset; + const char *name; + void *value; + RRDVAR_FLAGS flags :16; + RRDVAR_TYPE type:8; +}; - return rs; +static void rrdsetvar_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdsetvar, void *constructor_data) { + RRDSETVAR *rs = rrdsetvar; + struct rrdsetvar_constructor *ctr = constructor_data; + + ctr->flags &= ~RRDVAR_OPTIONS_REMOVED_ON_NEW_OBJECTS; + + rs->name = string_strdupz(ctr->name); + rs->type = ctr->type; + rs->flags = ctr->flags; + rrdsetvar_set_value_unsafe(rs, ctr->value); + + // create the rrdvariables while we are having a write lock to the dictionary + rrdsetvar_update_rrdvars_unsafe(ctr->rrdset, rs); +} + +static bool rrdsetvar_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdsetvar, void *new_rrdsetvar __maybe_unused, void *constructor_data) { + RRDSETVAR *rs = rrdsetvar; + struct rrdsetvar_constructor *ctr = constructor_data; + + ctr->flags &= ~RRDVAR_OPTIONS_REMOVED_ON_NEW_OBJECTS; + + RRDVAR_FLAGS options = rs->flags; + options &= ~RRDVAR_OPTIONS_REMOVED_ON_NEW_OBJECTS; + + if(((ctr->value == NULL && rs->value != NULL && rs->flags & RRDVAR_FLAG_ALLOCATED) || (rs->value == ctr->value)) + && ctr->flags == options && rs->type == ctr->type) { + // don't reset it - everything is the same, or as it should... + return false; + } + + internal_error(true, "RRDSETVAR: resetting variable '%s' of chart '%s' of host '%s', options from 0x%x to 0x%x, type from %d to %d", + string2str(rs->name), rrdset_id(ctr->rrdset), rrdhost_hostname(ctr->rrdset->rrdhost), + options, ctr->flags, rs->type, ctr->type); + + rrdsetvar_free_value_unsafe(rs); // we are going to change the options, so free it before setting it + rs->flags = ctr->flags; + rs->type = ctr->type; + rrdsetvar_set_value_unsafe(rs, ctr->value); + + // recreate the rrdvariables while we are having a write lock to the dictionary + rrdsetvar_update_rrdvars_unsafe(ctr->rrdset, rs); + return true; +} + +static void rrdsetvar_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdsetvar, void *rrdset __maybe_unused) { + RRDSET *st = rrdset; + RRDSETVAR *rs = rrdsetvar; + + rrdsetvar_free_rrdvars_unsafe(st, rs); + rrdsetvar_free_value_unsafe(rs); + string_freez(rs->name); + rs->name = NULL; +} + +void rrdsetvar_index_init(RRDSET *st) { + if(!st->rrdsetvar_root_index) { + st->rrdsetvar_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(st->rrdsetvar_root_index, rrdsetvar_insert_callback, NULL); + dictionary_register_conflict_callback(st->rrdsetvar_root_index, rrdsetvar_conflict_callback, NULL); + dictionary_register_delete_callback(st->rrdsetvar_root_index, rrdsetvar_delete_callback, st); + } +} + +void rrdsetvar_index_destroy(RRDSET *st) { + dictionary_destroy(st->rrdsetvar_root_index); + st->rrdsetvar_root_index = NULL; +} + +const RRDSETVAR_ACQUIRED *rrdsetvar_add_and_acquire(RRDSET *st, const char *name, RRDVAR_TYPE type, void *value, RRDVAR_FLAGS flags) { + struct rrdsetvar_constructor tmp = { + .name = name, + .type = type, + .value = value, + .flags = flags, + .rrdset = st, + }; + + const RRDSETVAR_ACQUIRED *rsa = (const RRDSETVAR_ACQUIRED *)dictionary_set_and_acquire_item_advanced(st->rrdsetvar_root_index, name, -1, NULL, sizeof(RRDSETVAR), &tmp); + return rsa; +} + +void rrdsetvar_add_and_leave_released(RRDSET *st, const char *name, RRDVAR_TYPE type, void *value, RRDVAR_FLAGS flags) { + const RRDSETVAR_ACQUIRED *rsa = rrdsetvar_add_and_acquire(st, name, type, value, flags); + dictionary_acquired_item_release(st->rrdsetvar_root_index, (const DICTIONARY_ITEM *)rsa); } void rrdsetvar_rename_all(RRDSET *st) { debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", rrdset_id(st), rrdset_name(st)); RRDSETVAR *rs; - for(rs = st->variables; rs ; rs = rs->next) - rrdsetvar_create_variables(rs); + dfe_start_write(st->rrdsetvar_root_index, rs) { + // should only be called while the rrdsetvar dict is write locked + rrdsetvar_update_rrdvars_unsafe(st, rs); + } + dfe_done(rs); - rrdsetcalc_link_matching(st); + rrdcalc_link_matching_alerts_to_rrdset(st); } -void rrdsetvar_free(RRDSETVAR *rs) { - RRDSET *st = rs->rrdset; - debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", rrdset_id(st), rrdset_name(st), string2str(rs->variable)); +void rrdsetvar_release_and_delete_all(RRDSET *st) { + RRDSETVAR *rs; + dfe_start_write(st->rrdsetvar_root_index, rs) { + dictionary_del_advanced(st->rrdsetvar_root_index, string2str(rs->name), (ssize_t)string_strlen(rs->name) + 1); + } + dfe_done(rs); +} - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->variables, rs, prev, next); - - rrdsetvar_free_variables(rs); - - string_freez(rs->variable); - - if(rs->options & RRDVAR_OPTION_ALLOCATED) - freez(rs->value); - - freez(rs); +void rrdsetvar_release(DICTIONARY *dict, const RRDSETVAR_ACQUIRED *rsa) { + dictionary_acquired_item_release(dict, (const DICTIONARY_ITEM *)rsa); } // -------------------------------------------------------------------------------------------------------------------- // custom chart variables -RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name) { - RRDHOST *host = st->rrdhost; - - STRING *n = rrdvar_name_to_string(name); - - rrdset_wrlock(st); - - // find it - RRDSETVAR *rs; - for(rs = st->variables; rs ; rs = rs->next) { - if(rs->variable == n) { - rrdset_unlock(st); - if(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) { - string_freez(n); - return rs; - } - else { - error("RRDSETVAR: custom variable '%s' on chart '%s' of host '%s', conflicts with an internal chart variable", string2str(n), rrdset_id(st), rrdhost_hostname(host)); - string_freez(n); - return NULL; - } - } - } - - // not found, allocate one - - NETDATA_DOUBLE *v = mallocz(sizeof(NETDATA_DOUBLE)); - *v = NAN; - - rs = rrdsetvar_create(st, string2str(n), RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR); - rrdset_unlock(st); - - string_freez(n); +const RRDSETVAR_ACQUIRED *rrdsetvar_custom_chart_variable_add_and_acquire(RRDSET *st, const char *name) { + STRING *name_string = rrdvar_name_to_string(name); + const RRDSETVAR_ACQUIRED *rs = rrdsetvar_add_and_acquire(st, string2str(name_string), RRDVAR_TYPE_CALCULATED, NULL, RRDVAR_FLAG_CUSTOM_CHART_VAR); + string_freez(name_string); return rs; } -void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, NETDATA_DOUBLE value) { - if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) || !(rs->options & RRDVAR_OPTION_ALLOCATED)) { +void rrdsetvar_custom_chart_variable_set(RRDSET *st, const RRDSETVAR_ACQUIRED *rsa, NETDATA_DOUBLE value) { + if(!rsa) return; + + RRDSETVAR *rs = dictionary_acquired_item_value((const DICTIONARY_ITEM *)rsa); + + if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->flags & RRDVAR_FLAG_CUSTOM_CHART_VAR) || !(rs->flags & RRDVAR_FLAG_ALLOCATED)) { error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " NETDATA_DOUBLE_FORMAT - " but the variable is not a custom chart one.", string2str(rs->variable), rrdset_id(rs->rrdset), rrdhost_hostname(rs->rrdset->rrdhost), value); + " but the variable is not a custom chart one (it has options 0x%x, value pointer %p). Ignoring request.", string2str(rs->name), rrdset_id(st), rrdhost_hostname(st->rrdhost), value, (uint32_t)rs->flags, rs->value); } else { NETDATA_DOUBLE *v = rs->value; @@ -172,7 +270,24 @@ void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, NETDATA_DOUBLE value) { *v = value; // mark the chart to be sent upstream - rrdset_flag_clear(rs->rrdset, RRDSET_FLAG_UPSTREAM_EXPOSED); + rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); } } } + +void rrdsetvar_print_to_streaming_custom_chart_variables(RRDSET *st, BUFFER *wb) { + // send the chart local custom variables + RRDSETVAR *rs; + dfe_start_read(st->rrdsetvar_root_index, rs) { + if(unlikely(rs->type == RRDVAR_TYPE_CALCULATED && rs->flags & RRDVAR_FLAG_CUSTOM_CHART_VAR)) { + NETDATA_DOUBLE *value = (NETDATA_DOUBLE *) rs->value; + + buffer_sprintf(wb + , "VARIABLE CHART %s = " NETDATA_DOUBLE_FORMAT "\n" + , string2str(rs->name) + , *value + ); + } + } + dfe_done(rs); +} diff --git a/database/rrdsetvar.h b/database/rrdsetvar.h index bcfa27aa63..86c4ef86e9 100644 --- a/database/rrdsetvar.h +++ b/database/rrdsetvar.h @@ -11,34 +11,20 @@ // This means, there will be no speed penalty for using // these variables -struct rrdsetvar { - STRING *variable; // variable name +extern void rrdsetvar_index_init(RRDSET *st); +extern void rrdsetvar_index_destroy(RRDSET *st); +extern void rrdsetvar_release_and_delete_all(RRDSET *st); - STRING *key_fullid; // chart type.chart id.variable - STRING *key_fullname; // chart type.chart name.variable +#define rrdsetvar_custom_chart_variable_release(st, rsa) rrdsetvar_release((st)->rrdsetvar_root_index, rsa) +extern void rrdsetvar_release(DICTIONARY *dict, const RRDSETVAR_ACQUIRED *rsa); - RRDVAR_TYPE type; - void *value; - - RRDVAR_OPTIONS options; - - RRDVAR *var_local; - RRDVAR *var_family; - RRDVAR *var_host; - RRDVAR *var_family_name; - RRDVAR *var_host_name; - - struct rrdset *rrdset; - - struct rrdsetvar *next; - struct rrdsetvar *prev; -}; - -extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name); -extern void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rv, NETDATA_DOUBLE value); +extern const RRDSETVAR_ACQUIRED *rrdsetvar_custom_chart_variable_add_and_acquire(RRDSET *st, const char *name); +extern void rrdsetvar_custom_chart_variable_set(RRDSET *st, const RRDSETVAR_ACQUIRED *rsa, NETDATA_DOUBLE value); extern void rrdsetvar_rename_all(RRDSET *st); -extern RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options); -extern void rrdsetvar_free(RRDSETVAR *rs); +extern const RRDSETVAR_ACQUIRED *rrdsetvar_add_and_acquire(RRDSET *st, const char *name, RRDVAR_TYPE type, void *value, RRDVAR_FLAGS flags); +extern void rrdsetvar_add_and_leave_released(RRDSET *st, const char *name, RRDVAR_TYPE type, void *value, RRDVAR_FLAGS flags); + +extern void rrdsetvar_print_to_streaming_custom_chart_variables(RRDSET *st, BUFFER *wb); #endif //NETDATA_RRDSETVAR_H diff --git a/database/rrdvar.c b/database/rrdvar.c index 43976951ac..28be4f6a1e 100644 --- a/database/rrdvar.c +++ b/database/rrdvar.c @@ -1,8 +1,19 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#define NETDATA_HEALTH_INTERNALS #include "rrd.h" +// the variables as stored in the variables indexes +// there are 3 indexes: +// 1. at each chart (RRDSET.rrdvar_root_index) +// 2. at each context (RRDFAMILY.rrdvar_root_index) +// 3. at each host (RRDHOST.rrdvar_root_index) +typedef struct rrdvar { + STRING *name; + void *value; + RRDVAR_FLAGS flags:24; + RRDVAR_TYPE type:8; +} RRDVAR; + // ---------------------------------------------------------------------------- // RRDVAR management @@ -20,41 +31,6 @@ inline int rrdvar_fix_name(char *variable) { return fixed; } -static inline RRDVAR *rrdvar_index_add(DICTIONARY *dict, RRDVAR *rv) { - return dictionary_set(dict, rrdvar_name(rv), rv, sizeof(RRDVAR)); -} - -static inline RRDVAR *rrdvar_index_del(DICTIONARY *dict, RRDVAR *rv) { - if(dictionary_del(dict, rrdvar_name(rv)) != 0) { - error("Request to remove RRDVAR '%s' from index failed. Not Found.", rrdvar_name(rv)); - return NULL; - } - - return rv; -} - -static inline RRDVAR *rrdvar_index_find(DICTIONARY *dict, STRING *name) { - return dictionary_get(dict, string2str(name)); -} - -inline void rrdvar_free(RRDHOST *host, DICTIONARY *dict, RRDVAR *rv) { - (void)host; - - if(!rv) return; - - if(dict) { - debug(D_VARIABLES, "Deleting variable '%s'", rrdvar_name(rv)); - if(unlikely(!rrdvar_index_del(dict, rv))) - error("RRDVAR: Attempted to delete variable '%s' from host '%s', but it is not found.", rrdvar_name(rv), rrdhost_hostname(host)); - } - - if(rv->options & RRDVAR_OPTION_ALLOCATED) - freez(rv->value); - - string_freez(rv->name); - freez(rv); -} - inline STRING *rrdvar_name_to_string(const char *name) { char *variable = strdupz(name); rrdvar_fix_name(variable); @@ -63,99 +39,145 @@ inline STRING *rrdvar_name_to_string(const char *name) { return name_string; } -inline RRDVAR *rrdvar_create_and_index(const char *scope __maybe_unused, DICTIONARY *dict, STRING *name, - RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value) { +struct rrdvar_constructor { + STRING *name; + void *value; + RRDVAR_FLAGS options:16; + RRDVAR_TYPE type:8; - RRDVAR *rv = rrdvar_index_find(dict, name); - if(unlikely(!rv)) { - debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", string2str(name), scope); + enum { + RRDVAR_REACT_NONE = 0, + RRDVAR_REACT_NEW = (1 << 0), + } react_action; +}; - rv = callocz(1, sizeof(RRDVAR)); - rv->name = string_dup(name); - rv->type = type; - rv->options = options; - rv->value = value; - rv->last_updated = now_realtime_sec(); +static void rrdvar_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdvar, void *constructor_data) { + RRDVAR *rv = rrdvar; + struct rrdvar_constructor *ctr = constructor_data; - RRDVAR *ret = rrdvar_index_add(dict, rv); - if(unlikely(ret != rv)) { - debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", string2str(name), scope); - freez(rv); - rv = NULL; - } - else - debug(D_VARIABLES, "Variable '%s' created in scope '%s'", string2str(name), scope); + ctr->options &= ~RRDVAR_OPTIONS_REMOVED_ON_NEW_OBJECTS; + + rv->name = string_dup(ctr->name); + rv->type = ctr->type; + rv->flags = ctr->options; + + if(!ctr->value) { + NETDATA_DOUBLE *v = mallocz(sizeof(NETDATA_DOUBLE)); + *v = NAN; + rv->value = v; + rv->flags |= RRDVAR_FLAG_ALLOCATED; } - else { - debug(D_VARIABLES, "Variable '%s' is already found in scope '%s'.", string2str(name), scope); + else + rv->value = ctr->value; - // this is important - // it must return NULL - not the existing variable - or double-free will happen - rv = NULL; - } - - return rv; + ctr->react_action = RRDVAR_REACT_NEW; } -void rrdvar_free_remaining_variables(RRDHOST *host, DICTIONARY *dict) { - RRDVAR *rv; - dfe_start_reentrant(dict, rv) { - rrdvar_free(host, dict, rv); - } - dfe_done(rv); +static void rrdvar_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdvar, void *nothing __maybe_unused) { + RRDVAR *rv = rrdvar; + + if(rv->flags & RRDVAR_FLAG_ALLOCATED) + freez(rv->value); + + string_freez(rv->name); + rv->name = NULL; } +DICTIONARY *rrdvariables_create(void) { + DICTIONARY *dict = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(dict, rrdvar_insert_callback, NULL); + dictionary_register_delete_callback(dict, rrdvar_delete_callback, NULL); + + return dict; +} + +void rrdvariables_destroy(DICTIONARY *dict) { + dictionary_destroy(dict); +} + +static inline const RRDVAR_ACQUIRED *rrdvar_get_and_acquire(DICTIONARY *dict, STRING *name) { + return (const RRDVAR_ACQUIRED *)dictionary_get_and_acquire_item_advanced(dict, string2str(name), (ssize_t)string_strlen(name) + 1); +} + +inline void rrdvar_release_and_del(DICTIONARY *dict, const RRDVAR_ACQUIRED *rva) { + if(unlikely(!dict || !rva)) return; + + RRDVAR *rv = dictionary_acquired_item_value((const DICTIONARY_ITEM *)rva); + + dictionary_del_advanced(dict, string2str(rv->name), (ssize_t)string_strlen(rv->name) + 1); + + dictionary_acquired_item_release(dict, (const DICTIONARY_ITEM *)rva); +} + +inline const RRDVAR_ACQUIRED *rrdvar_add_and_acquire(const char *scope __maybe_unused, DICTIONARY *dict, STRING *name, RRDVAR_TYPE type, RRDVAR_FLAGS options, void *value) { + if(unlikely(!dict || !name)) return NULL; + + struct rrdvar_constructor tmp = { + .name = name, + .value = value, + .type = type, + .options = options, + .react_action = RRDVAR_REACT_NONE, + }; + return (const RRDVAR_ACQUIRED *)dictionary_set_and_acquire_item_advanced(dict, string2str(name), (ssize_t)string_strlen(name) + 1, NULL, sizeof(RRDVAR), &tmp); +} + +void rrdvar_delete_all(DICTIONARY *dict) { + dictionary_flush(dict); +} + + // ---------------------------------------------------------------------------- // CUSTOM HOST VARIABLES -inline int rrdvar_walkthrough_read(DICTIONARY *dict, int (*callback)(const char *name, void *rrdvar, void *data), void *data) { +inline int rrdvar_walkthrough_read(DICTIONARY *dict, int (*callback)(const DICTIONARY_ITEM *item, void *rrdvar, void *data), void *data) { + if(unlikely(!dict)) return 0; // when health is not enabled return dictionary_walkthrough_read(dict, callback, data); } -static RRDVAR *rrdvar_custom_variable_create(const char *scope, DICTIONARY *dict, const char *name) { - NETDATA_DOUBLE *v = callocz(1, sizeof(NETDATA_DOUBLE)); - *v = NAN; +const RRDVAR_ACQUIRED *rrdvar_custom_host_variable_add_and_acquire(RRDHOST *host, const char *name) { + DICTIONARY *dict = host->rrdvars; + if(unlikely(!dict)) return NULL; // when health is not enabled STRING *name_string = rrdvar_name_to_string(name); - RRDVAR *rv = rrdvar_create_and_index(scope, dict, name_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v); - if(unlikely(!rv)) { - freez(v); - debug(D_VARIABLES, "Requested variable '%s' already exists - possibly 2 plugins are updating it at the same time.", string2str(name_string)); - - // find the existing one to return it - rv = rrdvar_index_find(dict, name_string); - } + const RRDVAR_ACQUIRED *rva = rrdvar_add_and_acquire("host", dict, name_string, RRDVAR_TYPE_CALCULATED, RRDVAR_FLAG_CUSTOM_HOST_VAR, NULL); string_freez(name_string); - - return rv; + return rva; } -RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) { - return rrdvar_custom_variable_create("host", host->rrdvar_root_index, name); -} +void rrdvar_custom_host_variable_set(RRDHOST *host, const RRDVAR_ACQUIRED *rva, NETDATA_DOUBLE value) { + if(unlikely(!host->rrdvars || !rva)) return; // when health is not enabled -void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value) { - if(rv->type != RRDVAR_TYPE_CALCULATED || !(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR) || !(rv->options & RRDVAR_OPTION_ALLOCATED)) - error("requested to set variable '%s' to value " NETDATA_DOUBLE_FORMAT " but the variable is not a custom one.", rrdvar_name(rv), value); + if(rrdvar_type(rva) != RRDVAR_TYPE_CALCULATED || !(rrdvar_flags(rva) & (RRDVAR_FLAG_CUSTOM_HOST_VAR | RRDVAR_FLAG_ALLOCATED))) + error("requested to set variable '%s' to value " NETDATA_DOUBLE_FORMAT " but the variable is not a custom one.", rrdvar_name(rva), value); else { + RRDVAR *rv = dictionary_acquired_item_value((const DICTIONARY_ITEM *)rva); NETDATA_DOUBLE *v = rv->value; if(*v != value) { *v = value; - rv->last_updated = now_realtime_sec(); - // if the host is streaming, send this variable upstream immediately - rrdpush_sender_send_this_host_variable_now(host, rv); + rrdpush_sender_send_this_host_variable_now(host, rva); } } } +void rrdvar_release(DICTIONARY *dict, const RRDVAR_ACQUIRED *rva) { + if(unlikely(!dict || !rva)) return; // when health is not enabled + dictionary_acquired_item_release(dict, (const DICTIONARY_ITEM *)rva); +} + // ---------------------------------------------------------------------------- // RRDVAR lookup -NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) { +NETDATA_DOUBLE rrdvar2number(const RRDVAR_ACQUIRED *rva) { + if(unlikely(!rva)) return NAN; + + RRDVAR *rv = dictionary_acquired_item_value((const DICTIONARY_ITEM *)rva); + switch(rv->type) { case RRDVAR_TYPE_CALCULATED: { NETDATA_DOUBLE *n = (NETDATA_DOUBLE *)rv->value; @@ -164,17 +186,17 @@ NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) { case RRDVAR_TYPE_TIME_T: { time_t *n = (time_t *)rv->value; - return *n; + return (NETDATA_DOUBLE)*n; } case RRDVAR_TYPE_COLLECTED: { collected_number *n = (collected_number *)rv->value; - return *n; + return (NETDATA_DOUBLE)*n; } case RRDVAR_TYPE_TOTAL: { total_number *n = (total_number *)rv->value; - return *n; + return (NETDATA_DOUBLE)*n; } case RRDVAR_TYPE_INT: { @@ -193,23 +215,26 @@ int health_variable_lookup(STRING *variable, RRDCALC *rc, NETDATA_DOUBLE *result if(!st) return 0; RRDHOST *host = st->rrdhost; - RRDVAR *rv; + const RRDVAR_ACQUIRED *rva; - rv = rrdvar_index_find(st->rrdvar_root_index, variable); - if(rv) { - *result = rrdvar2number(rv); + rva = rrdvar_get_and_acquire(st->rrdvars, variable); + if(rva) { + *result = rrdvar2number(rva); + dictionary_acquired_item_release(st->rrdvars, (const DICTIONARY_ITEM *)rva); return 1; } - rv = rrdvar_index_find(st->rrdfamily->rrdvar_root_index, variable); - if(rv) { - *result = rrdvar2number(rv); + rva = rrdvar_get_and_acquire(rrdfamily_rrdvars_dict(st->rrdfamily), variable); + if(rva) { + *result = rrdvar2number(rva); + dictionary_acquired_item_release(rrdfamily_rrdvars_dict(st->rrdfamily), (const DICTIONARY_ITEM *)rva); return 1; } - rv = rrdvar_index_find(host->rrdvar_root_index, variable); - if(rv) { - *result = rrdvar2number(rv); + rva = rrdvar_get_and_acquire(host->rrdvars, variable); + if(rva) { + *result = rrdvar2number(rva); + dictionary_acquired_item_release(host->rrdvars, (const DICTIONARY_ITEM *)rva); return 1; } @@ -222,19 +247,19 @@ int health_variable_lookup(STRING *variable, RRDCALC *rc, NETDATA_DOUBLE *result struct variable2json_helper { BUFFER *buf; size_t counter; - RRDVAR_OPTIONS options; + RRDVAR_FLAGS options; }; -static int single_variable2json(const char *name __maybe_unused, void *entry, void *data) { - struct variable2json_helper *helper = (struct variable2json_helper *)data; - RRDVAR *rv = (RRDVAR *)entry; - NETDATA_DOUBLE value = rrdvar2number(rv); +static int single_variable2json_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry __maybe_unused, void *helper_data) { + struct variable2json_helper *helper = (struct variable2json_helper *)helper_data; + const RRDVAR_ACQUIRED *rva = (const RRDVAR_ACQUIRED *)item; + NETDATA_DOUBLE value = rrdvar2number(rva); - if (helper->options == RRDVAR_OPTION_DEFAULT || rv->options & helper->options) { + if (helper->options == RRDVAR_FLAG_NONE || rrdvar_flags(rva) & helper->options) { if(unlikely(isnan(value) || isinf(value))) - buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rrdvar_name(rv)); + buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rrdvar_name(rva)); else - buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" NETDATA_DOUBLE_MODIFIER, helper->counter?",":"", rrdvar_name(rv), (NETDATA_DOUBLE)value); + buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" NETDATA_DOUBLE_MODIFIER, helper->counter?",":"", rrdvar_name(rva), (NETDATA_DOUBLE)value); helper->counter++; } @@ -246,11 +271,10 @@ void health_api_v1_chart_custom_variables2json(RRDSET *st, BUFFER *buf) { struct variable2json_helper helper = { .buf = buf, .counter = 0, - .options = RRDVAR_OPTION_CUSTOM_CHART_VAR - }; + .options = RRDVAR_FLAG_CUSTOM_CHART_VAR}; buffer_sprintf(buf, "{"); - rrdvar_walkthrough_read(st->rrdvar_root_index, single_variable2json, &helper); + rrdvar_walkthrough_read(st->rrdvars, single_variable2json_callback, &helper); buffer_strcat(buf, "\n\t\t\t}"); } @@ -260,20 +284,34 @@ void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf) { struct variable2json_helper helper = { .buf = buf, .counter = 0, - .options = RRDVAR_OPTION_DEFAULT - }; + .options = RRDVAR_FLAG_NONE}; buffer_sprintf(buf, "{\n\t\"chart\": \"%s\",\n\t\"chart_name\": \"%s\",\n\t\"chart_context\": \"%s\",\n\t\"chart_variables\": {", rrdset_id(st), rrdset_name(st), rrdset_context(st)); - rrdvar_walkthrough_read(st->rrdvar_root_index, single_variable2json, &helper); + rrdvar_walkthrough_read(st->rrdvars, single_variable2json_callback, &helper); buffer_sprintf(buf, "\n\t},\n\t\"family\": \"%s\",\n\t\"family_variables\": {", rrdset_family(st)); helper.counter = 0; - rrdvar_walkthrough_read(st->rrdfamily->rrdvar_root_index, single_variable2json, &helper); + rrdvar_walkthrough_read(rrdfamily_rrdvars_dict(st->rrdfamily), single_variable2json_callback, &helper); buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",\n\t\"host_variables\": {", rrdhost_hostname(host)); helper.counter = 0; - rrdvar_walkthrough_read(host->rrdvar_root_index, single_variable2json, &helper); + rrdvar_walkthrough_read(host->rrdvars, single_variable2json_callback, &helper); buffer_strcat(buf, "\n\t}\n}\n"); } +// ---------------------------------------------------------------------------- +// RRDVAR private members examination + +const char *rrdvar_name(const RRDVAR_ACQUIRED *rva) { + return dictionary_acquired_item_name((const DICTIONARY_ITEM *)rva); +} + +RRDVAR_FLAGS rrdvar_flags(const RRDVAR_ACQUIRED *rva) { + RRDVAR *rv = dictionary_acquired_item_value((const DICTIONARY_ITEM *)rva); + return rv->flags; +} +RRDVAR_TYPE rrdvar_type(const RRDVAR_ACQUIRED *rva) { + RRDVAR *rv = dictionary_acquired_item_value((const DICTIONARY_ITEM *)rva); + return rv->type; +} diff --git a/database/rrdvar.h b/database/rrdvar.h index 4ea367dbb7..8a93d02f0d 100644 --- a/database/rrdvar.h +++ b/database/rrdvar.h @@ -11,35 +11,32 @@ typedef enum rrdvar_type { RRDVAR_TYPE_COLLECTED = 3, RRDVAR_TYPE_TOTAL = 4, RRDVAR_TYPE_INT = 5 + + // this is 8 bit + // to increase it you have to set change the bitfield in + // rrdvar, rrdsetvar, rrddimvar } RRDVAR_TYPE; typedef enum rrdvar_options { - RRDVAR_OPTION_DEFAULT = 0, - RRDVAR_OPTION_ALLOCATED = (1 << 0), // the value ptr is allocated (not a reference) - RRDVAR_OPTION_CUSTOM_HOST_VAR = (1 << 1), // this is a custom host variable, not associated with a dimension - RRDVAR_OPTION_CUSTOM_CHART_VAR = (1 << 2), // this is a custom chart variable, not associated with a dimension - RRDVAR_OPTION_RRDCALC_LOCAL_VAR = (1 << 3), // this is a an alarm variable, attached to a chart - RRDVAR_OPTION_RRDCALC_FAMILY_VAR = (1 << 4), // this is a an alarm variable, attached to a family - RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR = (1 << 5), // this is a an alarm variable, attached to the host, using the chart id - RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR = (1 << 6), // this is a an alarm variable, attached to the host, using the chart name -} RRDVAR_OPTIONS; + RRDVAR_FLAG_NONE = 0, + RRDVAR_FLAG_ALLOCATED = (1 << 0), // the value ptr is allocated (not a reference) + RRDVAR_FLAG_CUSTOM_HOST_VAR = (1 << 1), // this is a custom host variable, not associated with a dimension + RRDVAR_FLAG_CUSTOM_CHART_VAR = (1 << 2), // this is a custom chart variable, not associated with a dimension + RRDVAR_FLAG_RRDCALC_LOCAL_VAR = (1 << 3), // this is a an alarm variable, attached to a chart + RRDVAR_FLAG_RRDCALC_FAMILY_VAR = (1 << 4), // this is a an alarm variable, attached to a family + RRDVAR_FLAG_RRDCALC_HOST_CHARTID_VAR = (1 << 5), // this is a an alarm variable, attached to the host, using the chart id + RRDVAR_FLAG_RRDCALC_HOST_CHARTNAME_VAR = (1 << 6), // this is a an alarm variable, attached to the host, using the chart name -// the variables as stored in the variables indexes -// there are 3 indexes: -// 1. at each chart (RRDSET.rrdvar_root_index) -// 2. at each context (RRDFAMILY.rrdvar_root_index) -// 3. at each host (RRDHOST.rrdvar_root_index) -struct rrdvar { - STRING *name; + // this is 24 bit + // to increase it you have to set change the bitfield in + // rrdvar, rrdsetvar, rrddimvar +} RRDVAR_FLAGS; - void *value; - time_t last_updated; +#define RRDVAR_OPTIONS_REMOVED_ON_NEW_OBJECTS \ + (RRDVAR_FLAG_ALLOCATED) - RRDVAR_OPTIONS options:16; - RRDVAR_TYPE type:8; -}; - -#define rrdvar_name(rv) string2str((rv)->name) +#define RRDVAR_OPTIONS_REMOVED_WHEN_PROPAGATING_TO_RRDVAR \ + (RRDVAR_FLAG_ALLOCATED) #define RRDVAR_MAX_LENGTH 1024 @@ -49,15 +46,26 @@ extern int rrdvar_fix_name(char *variable); extern STRING *rrdvar_name_to_string(const char *name); -extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name); -extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value); -extern void rrdvar_free_remaining_variables(RRDHOST *host, DICTIONARY *dict); +extern const RRDVAR_ACQUIRED *rrdvar_custom_host_variable_add_and_acquire(RRDHOST *host, const char *name); +extern void rrdvar_custom_host_variable_set(RRDHOST *host, const RRDVAR_ACQUIRED *rva, NETDATA_DOUBLE value); -extern int rrdvar_walkthrough_read(DICTIONARY *dict, int (*callback)(const char *name, void *rrdvar, void *data), void *data); +extern int rrdvar_walkthrough_read(DICTIONARY *dict, int (*callback)(const DICTIONARY_ITEM *item, void *rrdvar, void *data), void *data); -extern NETDATA_DOUBLE rrdvar2number(RRDVAR *rv); +#define rrdvar_custom_host_variable_release(host, rva) rrdvar_release((host)->rrdvars, rva) +extern void rrdvar_release(DICTIONARY *dict, const RRDVAR_ACQUIRED *rva); -extern RRDVAR *rrdvar_create_and_index(const char *scope, DICTIONARY *dict, STRING *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value); -extern void rrdvar_free(RRDHOST *host, DICTIONARY *dict, RRDVAR *rv); +extern NETDATA_DOUBLE rrdvar2number(const RRDVAR_ACQUIRED *rva); + +extern const RRDVAR_ACQUIRED *rrdvar_add_and_acquire(const char *scope, DICTIONARY *dict, STRING *name, RRDVAR_TYPE type, RRDVAR_FLAGS options, void *value); +extern void rrdvar_release_and_del(DICTIONARY *dict, const RRDVAR_ACQUIRED *rva); + +extern DICTIONARY *rrdvariables_create(void); +extern void rrdvariables_destroy(DICTIONARY *dict); + +extern void rrdvar_delete_all(DICTIONARY *dict); + +extern const char *rrdvar_name(const RRDVAR_ACQUIRED *rva); +extern RRDVAR_FLAGS rrdvar_flags(const RRDVAR_ACQUIRED *rva); +extern RRDVAR_TYPE rrdvar_type(const RRDVAR_ACQUIRED *rva); #endif //NETDATA_RRDVAR_H diff --git a/database/sqlite/sqlite_aclk.c b/database/sqlite/sqlite_aclk.c index 33a8a077de..dca1ff793f 100644 --- a/database/sqlite/sqlite_aclk.c +++ b/database/sqlite/sqlite_aclk.c @@ -1023,9 +1023,6 @@ void sql_check_aclk_table_list(struct aclk_database_worker_config *wc) db_execute("DELETE FROM dimension_delete WHERE host_id NOT IN (SELECT host_id FROM host) " " OR unixepoch() - date_created > 604800;"); - db_execute("DELETE FROM chart_hash WHERE CAST(last_used AS INT) < unixepoch() - 604800;"); - db_execute("DELETE FROM chart_hash_map WHERE hash_id NOT IN (SELECT hash_id FROM chart_hash);"); - return; } diff --git a/database/sqlite/sqlite_aclk_chart.c b/database/sqlite/sqlite_aclk_chart.c index 882b8d83c8..26b9acfce9 100644 --- a/database/sqlite/sqlite_aclk_chart.c +++ b/database/sqlite/sqlite_aclk_chart.c @@ -150,7 +150,7 @@ int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_dat if (likely(claim_id)) { struct chart_instance_updated chart_payload; memset(&chart_payload, 0, sizeof(chart_payload)); - chart_payload.config_hash = get_str_from_uuid(&st->uuid); + chart_payload.config_hash = get_str_from_uuid(&st->hash_uuid); chart_payload.update_every = st->update_every; chart_payload.memory_mode = st->rrd_memory_mode; chart_payload.name = (char *)rrdset_name(st); @@ -164,7 +164,7 @@ int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_dat size_t size; char *payload = generate_chart_instance_updated(&size, &chart_payload); if (likely(payload)) - rc = aclk_add_chart_payload(wc, st->chart_uuid, claim_id, ACLK_PAYLOAD_CHART, (void *) payload, size, NULL, 1); + rc = aclk_add_chart_payload(wc, &st->chart_uuid, claim_id, ACLK_PAYLOAD_CHART, (void *) payload, size, NULL, 1); freez(payload); chart_instance_updated_destroy(&chart_payload); } @@ -588,24 +588,23 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl RRDHOST *host = wc->host; if (likely(host)) { - rrdhost_rdlock(host); RRDSET *st; - rrdset_foreach_read(st, host) - { - rrdset_rdlock(st); + rrdset_foreach_read(st, host) { rrdset_flag_clear(st, RRDSET_FLAG_ACLK); RRDDIM *rd; - rrddim_foreach_read(rd, st) - { + rrddim_foreach_read(rd, st) { rrddim_flag_clear(rd, RRDDIM_FLAG_ACLK); rd->aclk_live_status = (rd->aclk_live_status == 0); } - rrdset_unlock(st); + rrddim_foreach_done(rd); } - rrdhost_unlock(host); - } else + rrdset_foreach_done(st); + } + else error_report("ACLK synchronization thread for %s is not linked to HOST", wc->host_guid); - } else { + + } + else { log_access( "ACLK STA [%s (%s)]: RESTARTING CHART SYNC FROM SEQUENCE %" PRIu64, wc->node_id, @@ -1268,29 +1267,26 @@ void sql_check_chart_liveness(RRDSET *st) { if (unlikely(rrdset_is_ar_chart(st))) return; - rrdset_rdlock(st); - - if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) { - rrdset_unlock(st); + if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) return; - } if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) { - if (likely(st->dimensions && st->counter_done && !queue_chart_to_aclk(st))) { + if (likely(rrdset_number_of_dimensions(st) && st->counter_done && !queue_chart_to_aclk(st))) { debug(D_ACLK_SYNC,"Check chart liveness [%s] submit chart definition", rrdset_name(st)); rrdset_flag_set(st, RRDSET_FLAG_ACLK); } } else debug(D_ACLK_SYNC,"Check chart liveness [%s] chart definition already submitted", rrdset_name(st)); + time_t mark = now_realtime_sec(); debug(D_ACLK_SYNC,"Check chart liveness [%s] scanning dimensions", rrdset_name(st)); rrddim_foreach_read(rd, st) { - if (!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) + if (!rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)) queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, mark)); } - rrdset_unlock(st); + rrddim_foreach_done(rd); } // ST is read locked diff --git a/database/sqlite/sqlite_aclk_node.c b/database/sqlite/sqlite_aclk_node.c index 6a6372edd8..ed8996bac5 100644 --- a/database/sqlite/sqlite_aclk_node.c +++ b/database/sqlite/sqlite_aclk_node.c @@ -12,7 +12,6 @@ DICTIONARY *collectors_from_charts(RRDHOST *host, DICTIONARY *dict) { RRDSET *st; char name[500]; - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { struct collector_info col = { @@ -23,7 +22,7 @@ DICTIONARY *collectors_from_charts(RRDHOST *host, DICTIONARY *dict) { dictionary_set(dict, name, &col, sizeof(struct collector_info)); } } - rrdhost_unlock(host); + rrdset_foreach_done(st); return dict; } @@ -36,7 +35,7 @@ void sql_build_node_collectors(struct aclk_database_worker_config *wc) return; struct update_node_collectors upd_node_collectors; - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); upd_node_collectors.node_id = wc->node_id; upd_node_collectors.claim_id = get_agent_claimid(); diff --git a/database/sqlite/sqlite_db_migration.c b/database/sqlite/sqlite_db_migration.c index 7b2b16da4f..fb54954c51 100644 --- a/database/sqlite/sqlite_db_migration.c +++ b/database/sqlite/sqlite_db_migration.c @@ -64,6 +64,15 @@ const char *database_migrate_v2_v3[] = { NULL }; +const char *database_migrate_v4_v5[] = { + "DROP TABLE IF EXISTS chart_active;", + "DROP TABLE IF EXISTS dimension_active;", + "DROP TABLE IF EXISTS chart_hash;", + "DROP TABLE IF EXISTS chart_hash_map;", + "DROP VIEW IF EXISTS v_chart_hash;", + NULL +}; + static int do_migration_v1_v2(sqlite3 *database, const char *name) { UNUSED(name); @@ -116,6 +125,14 @@ static int do_migration_v3_v4(sqlite3 *database, const char *name) return 0; } +static int do_migration_v4_v5(sqlite3 *database, const char *name) +{ + UNUSED(name); + info("Running \"%s\" database migration", name); + + return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v4_v5[0]); +} + static int do_migration_noop(sqlite3 *database, const char *name) { UNUSED(database); @@ -163,6 +180,7 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = { {.name = "v1 to v2", .func = do_migration_v1_v2}, {.name = "v2 to v3", .func = do_migration_v2_v3}, {.name = "v3 to v4", .func = do_migration_v3_v4}, + {.name = "v4 to v5", .func = do_migration_v4_v5}, // the terminator of this array {.name = NULL, .func = NULL} }; diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index 6f71e3f6c3..31fd33602d 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -3,7 +3,7 @@ #include "sqlite_functions.h" #include "sqlite_db_migration.h" -#define DB_METADATA_VERSION 4 +#define DB_METADATA_VERSION 5 const char *database_config[] = { "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, " @@ -21,11 +21,6 @@ const char *database_config[] = { "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, " "multiplier int, divisor int , algorithm int, options text);", - "DROP TABLE IF EXISTS chart_active;", - "DROP TABLE IF EXISTS dimension_active;", - - "CREATE TABLE IF NOT EXISTS chart_active(chart_id blob PRIMARY KEY, date_created int);", - "CREATE TABLE IF NOT EXISTS dimension_active(dim_id blob primary key, date_created int);", "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int);", "CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);", "CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);", @@ -46,36 +41,16 @@ const char *database_config[] = { "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, " "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key));", - "CREATE TABLE IF NOT EXISTS chart_hash_map(chart_id blob , hash_id blob, UNIQUE (chart_id, hash_id));", - - "CREATE TABLE IF NOT EXISTS chart_hash(hash_id blob PRIMARY KEY,type text, id text, name text, " - "family text, context text, title text, unit text, plugin text, " - "module text, priority integer, chart_type, last_used);", - - "CREATE VIEW IF NOT EXISTS v_chart_hash as SELECT ch.*, chm.chart_id FROM chart_hash ch, chart_hash_map chm " - "WHERE ch.hash_id = chm.hash_id;", - "CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)" " SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END;", - "CREATE TRIGGER IF NOT EXISTS tr_v_chart_hash INSTEAD OF INSERT on v_chart_hash BEGIN " - "INSERT INTO chart_hash (hash_id, type, id, name, family, context, title, unit, plugin, " - "module, priority, chart_type, last_used) " - "values (new.hash_id, new.type, new.id, new.name, new.family, new.context, new.title, new.unit, new.plugin, " - "new.module, new.priority, new.chart_type, unixepoch()) " - "ON CONFLICT (hash_id) DO UPDATE SET last_used = unixepoch(); " - "INSERT INTO chart_hash_map (chart_id, hash_id) values (new.chart_id, new.hash_id) " - "on conflict (chart_id, hash_id) do nothing; END; ", - NULL }; const char *database_cleanup[] = { - "delete from chart where chart_id not in (select chart_id from dimension);", - "delete from host where host_id not in (select host_id from chart);", - "delete from chart_label where chart_id not in (select chart_id from chart);", - "DELETE FROM chart_hash_map WHERE chart_id NOT IN (SELECT chart_id FROM chart);", - "DELETE FROM chart_hash WHERE hash_id NOT IN (SELECT hash_id FROM chart_hash_map);", + "DELETE FROM chart WHERE chart_id NOT IN (SELECT chart_id FROM dimension);", + "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart);", + "DELETE FROM chart_label WHERE chart_id NOT IN (SELECT chart_id FROM chart);", "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);", "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host);", "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host);", @@ -190,88 +165,6 @@ int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement) return rc; } -/* - * Store a chart or dimension UUID in chart_active or dimension_active - * The statement that will be prepared determines that - */ - -static int store_active_uuid_object(sqlite3_stmt **res, char *statement, uuid_t *uuid) -{ - int rc; - - // Check if we should need to prepare the statement - if (!*res) { - rc = prepare_statement(db_meta, statement, res); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to store active object, rc = %d", rc); - return rc; - } - } - - rc = sqlite3_bind_blob(*res, 1, uuid, sizeof(*uuid), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to bind input parameter to store active object, rc = %d", rc); - else - rc = execute_insert(*res); - return rc; -} - -/* - * Marks a chart with UUID as active - * Input: UUID - */ -void store_active_chart(uuid_t *chart_uuid) -{ - static __thread sqlite3_stmt *res = NULL; - int rc; - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("Database has not been initialized"); - return; - } - - if (unlikely(!chart_uuid)) - return; - - rc = store_active_uuid_object(&res, SQL_STORE_ACTIVE_CHART, chart_uuid); - if (rc != SQLITE_DONE) - error_report("Failed to store active chart, rc = %d", rc); - - rc = sqlite3_reset(res); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to finalize statement in store active chart, rc = %d", rc); - return; -} - -/* - * Marks a dimension with UUID as active - * Input: UUID - */ -void store_active_dimension(uuid_t *dimension_uuid) -{ - static __thread sqlite3_stmt *res = NULL; - int rc; - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("Database has not been initialized"); - return; - } - - if (unlikely(!dimension_uuid)) - return; - - rc = store_active_uuid_object(&res, SQL_STORE_ACTIVE_DIMENSION, dimension_uuid); - if (rc != SQLITE_DONE) - error_report("Failed to store active dimension, rc = %d", rc); - - rc = sqlite3_reset(res); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to finalize statement in store active dimension, rc = %d", rc); - return; -} - static int check_table_integrity_cb(void *data, int argc, char **argv, char **column) { int *status = data; @@ -623,7 +516,7 @@ int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid) } } - rc = sqlite3_bind_blob(res, 1, st->chart_uuid, sizeof(*st->chart_uuid), SQLITE_STATIC); + rc = sqlite3_bind_blob(res, 1, &st->chart_uuid, sizeof(st->chart_uuid), SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; @@ -642,7 +535,7 @@ int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid) } else { uuid_generate(*store_uuid); - status = sql_store_dimension(store_uuid, st->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); + status = sql_store_dimension(store_uuid, &st->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); if (unlikely(status)) error_report("Failed to store dimension metadata in the database"); } @@ -697,20 +590,20 @@ bind_fail: * Do a database lookup to find the UUID of a chart * */ -uuid_t *find_chart_uuid(RRDHOST *host, const char *type, const char *id, const char *name) +int find_chart_uuid(RRDHOST *host, const char *type, const char *id, const char *name, uuid_t *store_uuid) { static __thread sqlite3_stmt *res = NULL; - uuid_t *uuid = NULL; int rc; + int status = 1; if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) - return NULL; + return 1; if (unlikely(!res)) { rc = prepare_statement(db_meta, SQL_FIND_CHART_UUID, &res); if (rc != SQLITE_OK) { error_report("Failed to prepare statement to lookup chart UUID in the database"); - return NULL; + return 1; } } @@ -731,32 +624,23 @@ uuid_t *find_chart_uuid(RRDHOST *host, const char *type, const char *id, const c goto bind_fail; rc = sqlite3_step_monitored(res); - if (likely(rc == SQLITE_ROW)) { - uuid = mallocz(sizeof(uuid_t)); - uuid_copy(*uuid, sqlite3_column_blob(res, 0)); + if (likely(rc == SQLITE_ROW && sqlite3_column_bytes(res,0) == sizeof(*store_uuid))) { + uuid_copy(*store_uuid, *((uuid_t *) sqlite3_column_blob(res, 0))); + status = 0; } rc = sqlite3_reset(res); if (unlikely(rc != SQLITE_OK)) error_report("Failed to reset statement when searching for a chart UUID, rc = %d", rc); -#ifdef NETDATA_INTERNAL_CHECKS - char uuid_str[GUID_LEN + 1]; - if (likely(uuid)) { - uuid_unparse_lower(*uuid, uuid_str); - debug(D_METADATALOG, "Found UUID %s for chart %s.%s", uuid_str, type, name ? name : id); - } - else - debug(D_METADATALOG, "UUID not found for chart %s.%s", type, name ? name : id); -#endif - return uuid; + return status; bind_fail: error_report("Failed to bind input parameter to perform chart UUID database lookup, rc = %d", rc); rc = sqlite3_reset(res); if (unlikely(rc != SQLITE_OK)) error_report("Failed to reset statement when searching for a chart UUID, rc = %d", rc); - return NULL; + return 1; } int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const char *name) @@ -767,7 +651,8 @@ int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const return 0; rc = sql_store_chart( - chart_uuid, &st->rrdhost->host_uuid, rrdset_type(st), id, name, + chart_uuid, &st->rrdhost->host_uuid, + rrdset_parts_type(st), id, name, rrdset_family(st), rrdset_context(st), rrdset_title(st), rrdset_units(st), rrdset_plugin_name(st), rrdset_module_name(st), st->priority, st->update_every, st->chart_type, @@ -776,28 +661,6 @@ int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const return rc; } -uuid_t *create_chart_uuid(RRDSET *st, const char *id, const char *name) -{ - uuid_t *uuid = NULL; - int rc; - - uuid = mallocz(sizeof(uuid_t)); - uuid_generate(*uuid); - -#ifdef NETDATA_INTERNAL_CHECKS - char uuid_str[GUID_LEN + 1]; - uuid_unparse_lower(*uuid, uuid_str); - debug(D_METADATALOG,"Generating uuid [%s] for chart %s under host %s", uuid_str, rrdset_id(st), rrdhost_hostname(st->rrdhost)); -#endif - - rc = update_chart_metadata(uuid, st, id, name); - - if (unlikely(rc)) - error_report("Failed to store chart metadata in the database"); - - return uuid; -} - static int exec_statement_with_uuid(const char *sql, uuid_t *uuid) { int rc, result = 1; @@ -1695,34 +1558,6 @@ skip_store: return rc != SQLITE_DONE; } -#define SQL_INS_CHART_LABEL "insert or replace into chart_label " \ - "(chart_id, source_type, label_key, label_value, date_created) " \ - "values (@chart, @source, @label, @value, unixepoch());" - -void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value) -{ - static __thread sqlite3_stmt *res = NULL; - int rc; - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("Database has not been initialized"); - return; - } - - if (unlikely(!res)) { - rc = prepare_statement(db_meta, SQL_INS_CHART_LABEL, &res); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement store chart labels"); - return; - } - } - - sql_store_label(res, chart_uuid, source_type, label, value); - - return; -} - #define SQL_INS_HOST_LABEL "INSERT OR REPLACE INTO host_label " \ "(host_id, source_type, label_key, label_value, date_created) " \ "values (@chart, @source, @label, @value, unixepoch());" @@ -1934,7 +1769,7 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para if (unlikely(!rd)) continue; if (sqlite3_column_int(res, 9) == 1) - rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN); + rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN); rd->next = (*param_list)->rd; (*param_list)->rd = rd; } @@ -1964,186 +1799,6 @@ failed: } -/* - * Store a chart hash in the database - */ - -#define SQL_STORE_CHART_HASH "insert into v_chart_hash (hash_id, type, id, " \ - "name, family, context, title, unit, plugin, module, priority, chart_type, last_used, chart_id) " \ - "values (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11, ?12, unixepoch(), ?13);" - -int sql_store_chart_hash( - uuid_t *hash_id, uuid_t *chart_id, const char *type, const char *id, const char *name, const char *family, - const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority, - RRDSET_TYPE chart_type) -{ - static __thread sqlite3_stmt *res = NULL; - int rc, param = 0; - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) - return 0; - error_report("Database has not been initialized"); - return 1; - } - - if (unlikely(!res)) { - rc = prepare_statement(db_meta, SQL_STORE_CHART_HASH, &res); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to store chart, rc = %d", rc); - return 1; - } - } - - param++; - rc = sqlite3_bind_blob(res, 1, hash_id, sizeof(*hash_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_text(res, 2, type, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_text(res, 3, id, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - if (name && *name) - rc = sqlite3_bind_text(res, 4, name, -1, SQLITE_STATIC); - else - rc = sqlite3_bind_null(res, 4); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_text(res, 5, family, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_text(res, 6, context, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_text(res, 7, title, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_text(res, 8, units, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_text(res, 9, plugin, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_text(res, 10, module, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_int(res, 11, (int) priority); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_int(res, 12, chart_type); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - param++; - rc = sqlite3_bind_blob(res, 13, chart_id, sizeof(*chart_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - - rc = execute_insert(res); - if (unlikely(rc != SQLITE_DONE)) - error_report("Failed to store chart hash_id, rc = %d", rc); - - rc = sqlite3_reset(res); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to reset statement in chart hash_id store function, rc = %d", rc); - - return 0; - - bind_fail: - error_report("Failed to bind parameter %d to store chart hash_id, rc = %d", param, rc); - rc = sqlite3_reset(res); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to reset statement in chart hash_id store function, rc = %d", rc); - return 1; -} - -/* - chart hashes are used for cloud communication. - if cloud is disabled or openssl is not available (which will prevent cloud connectivity) - skip hash calculations -*/ -void compute_chart_hash(RRDSET *st) -{ -#if !defined DISABLE_CLOUD && defined ENABLE_HTTPS - EVP_MD_CTX *evpctx; - unsigned char hash_value[EVP_MAX_MD_SIZE]; - unsigned int hash_len; - char priority_str[32]; - - if (rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) { - internal_error(true, "Skipping compute_chart_hash for host %s because context streaming is enabled", rrdhost_hostname(st->rrdhost)); - return; - } - - sprintf(priority_str, "%ld", st->priority); - - evpctx = EVP_MD_CTX_create(); - EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL); - //EVP_DigestUpdate(evpctx, st->type, strlen(st->type)); - EVP_DigestUpdate(evpctx, rrdset_id(st), string_strlen(st->id)); - EVP_DigestUpdate(evpctx, rrdset_name(st), string_strlen(st->name)); - EVP_DigestUpdate(evpctx, rrdset_family(st), string_strlen(st->family)); - EVP_DigestUpdate(evpctx, rrdset_context(st), string_strlen(st->context)); - EVP_DigestUpdate(evpctx, rrdset_title(st), string_strlen(st->title)); - EVP_DigestUpdate(evpctx, rrdset_units(st), string_strlen(st->units)); - EVP_DigestUpdate(evpctx, rrdset_plugin_name(st), string_strlen(st->plugin_name)); - EVP_DigestUpdate(evpctx, rrdset_module_name(st), string_strlen(st->module_name)); -// EVP_DigestUpdate(evpctx, priority_str, strlen(priority_str)); - EVP_DigestUpdate(evpctx, &st->priority, sizeof(st->priority)); - EVP_DigestUpdate(evpctx, &st->chart_type, sizeof(st->chart_type)); - EVP_DigestFinal_ex(evpctx, hash_value, &hash_len); - EVP_MD_CTX_destroy(evpctx); - fatal_assert(hash_len > sizeof(uuid_t)); - - char uuid_str[GUID_LEN + 1]; - uuid_unparse_lower(*((uuid_t *) &hash_value), uuid_str); - //info("Calculating HASH %s for chart %s", uuid_str, st->name); - uuid_copy(st->uuid, *((uuid_t *) &hash_value)); - - (void)sql_store_chart_hash( - (uuid_t *)&hash_value, - st->chart_uuid, - rrdset_type(st), - rrdset_id(st), - rrdset_name(st), - rrdset_family(st), - rrdset_context(st), - rrdset_title(st), - rrdset_units(st), - rrdset_plugin_name(st), - rrdset_module_name(st), - st->priority, - st->chart_type); -#else - UNUSED(st); -#endif - return; -} - #define SQL_STORE_CLAIM_ID "insert into node_instance " \ "(host_id, claim_id, date_created) values (@host_id, @claim_id, unixepoch()) " \ "on conflict(host_id) do update set claim_id = excluded.claim_id;" diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h index 1e4ba43629..557997a805 100644 --- a/database/sqlite/sqlite_functions.h +++ b/database/sqlite/sqlite_functions.h @@ -37,18 +37,12 @@ typedef enum db_check_action_type { #define SQL_FIND_CHART_UUID \ "select chart_id from chart where host_id = @host and type=@type and id=@id and (name is null or name=@name) and chart_id is not null;" -#define SQL_STORE_ACTIVE_CHART \ - "insert or replace into chart_active (chart_id, date_created) values (@id, unixepoch());" - #define SQL_STORE_DIMENSION \ "INSERT OR REPLACE into dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm) values (?0001,?0002,?0003,?0004,?0005,?0006,?0007);" #define SQL_FIND_DIMENSION_UUID \ "select dim_id from dimension where chart_id=@chart and id=@id and name=@name and length(dim_id)=16;" -#define SQL_STORE_ACTIVE_DIMENSION \ - "insert or replace into dimension_active (dim_id, date_created) values (@id, unixepoch());" - #define CHECK_SQLITE_CONNECTION(db_meta) \ if (unlikely(!db_meta)) { \ if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) { \ @@ -83,12 +77,9 @@ extern int sql_store_dimension(uuid_t *dim_uuid, uuid_t *chart_uuid, const char collected_number divisor, int algorithm); extern int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid); -extern void store_active_dimension(uuid_t *dimension_uuid); -extern uuid_t *find_chart_uuid(RRDHOST *host, const char *type, const char *id, const char *name); -extern uuid_t *create_chart_uuid(RRDSET *st, const char *id, const char *name); +extern int find_chart_uuid(RRDHOST *host, const char *type, const char *id, const char *name, uuid_t *store_uuid); extern int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const char *name); -extern void store_active_chart(uuid_t *dimension_uuid); extern int find_uuid_type(uuid_t *uuid); @@ -103,7 +94,6 @@ extern void add_migrated_file(char *path, uint64_t file_size); extern void db_unlock(void); extern void db_lock(void); extern void delete_dimension_uuid(uuid_t *dimension_uuid); -extern void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value); extern void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart); extern void store_claim_id(uuid_t *host_id, uuid_t *claim_id); extern int update_node_id(uuid_t *host_id, uuid_t *node_id); @@ -112,7 +102,6 @@ extern int get_host_id(uuid_t *node_id, uuid_t *host_id); extern void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id); extern struct node_instance_list *get_node_list(void); extern void sql_load_node_id(RRDHOST *host); -extern void compute_chart_hash(RRDSET *st); extern int sql_set_dimension_option(uuid_t *dim_uuid, char *option); char *get_hostname_by_node_id(char *node_id); void free_temporary_host(RRDHOST *host); diff --git a/database/sqlite/sqlite_health.c b/database/sqlite/sqlite_health.c index 58ee73b543..2ac50867b4 100644 --- a/database/sqlite/sqlite_health.c +++ b/database/sqlite/sqlite_health.c @@ -632,12 +632,15 @@ void sql_health_alarm_log_load(RRDHOST *host) { return; } - netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); - - DICTIONARY *all_rrdcalcs = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + DICTIONARY *all_rrdcalcs = dictionary_create( + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) + foreach_rrdcalc_in_rrdhost_read(host, rc) { dictionary_set(all_rrdcalcs, rrdcalc_name(rc), rc, sizeof(*rc)); + } + foreach_rrdcalc_in_rrdhost_done(rc); + + netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); while (sqlite3_step_monitored(res) == SQLITE_ROW) { ALARM_ENTRY *ae = NULL; @@ -790,11 +793,11 @@ void sql_health_alarm_log_load(RRDHOST *host) { loaded++; } + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); + dictionary_destroy(all_rrdcalcs); all_rrdcalcs = NULL; - netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); - if(!host->health_max_unique_id) host->health_max_unique_id = (uint32_t)now_realtime_sec(); if(!host->health_max_alarm_id) host->health_max_alarm_id = (uint32_t)now_realtime_sec(); diff --git a/exporting/json/json.c b/exporting/json/json.c index 06100ba5e2..dc2d5aae99 100644 --- a/exporting/json/json.c +++ b/exporting/json/json.c @@ -197,7 +197,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM rrdset_name(st), rrdset_family(st), rrdset_context(st), - rrdset_type(st), + rrdset_parts_type(st), rrdset_units(st), rrddim_id(rd), rrddim_name(rd), @@ -281,7 +281,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd rrdset_name(st), rrdset_family(st), rrdset_context(st), - rrdset_type(st), + rrdset_parts_type(st), rrdset_units(st), rrddim_id(rd), rrddim_name(rd), diff --git a/exporting/process_data.c b/exporting/process_data.c index e1e939c51d..05cf0a98bb 100644 --- a/exporting/process_data.c +++ b/exporting/process_data.c @@ -338,26 +338,22 @@ void prepare_buffers(struct engine *engine) rrd_rdlock(); RRDHOST *host; - rrdhost_foreach_read(host) - { - rrdhost_rdlock(host); + rrdhost_foreach_read(host) { start_host_formatting(engine, host); RRDSET *st; - rrdset_foreach_read(st, host) - { - rrdset_rdlock(st); + rrdset_foreach_read(st, host) { start_chart_formatting(engine, st); RRDDIM *rd; rrddim_foreach_read(rd, st) metric_formatting(engine, rd); + rrddim_foreach_done(rd); end_chart_formatting(engine, st); - rrdset_unlock(st); } + rrdset_foreach_done(st); variables_formatting(engine, host); end_host_formatting(engine, host); - rrdhost_unlock(host); } rrd_unlock(); netdata_thread_enable_cancelability(); diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c index 26c0811d6f..5719db4187 100644 --- a/exporting/prometheus/prometheus.c +++ b/exporting/prometheus/prometheus.c @@ -349,12 +349,12 @@ struct host_variables_callback_options { * @param data callback options. * @return Returns 1 if the chart can be sent, 0 otherwise. */ -static int print_host_variables(const char *name __maybe_unused, void *rv_ptr, void *data) { - RRDVAR *rv = rv_ptr; +static int print_host_variables_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rv_ptr __maybe_unused, void *data) { + const RRDVAR_ACQUIRED *rv = (const RRDVAR_ACQUIRED *)item; struct host_variables_callback_options *opts = data; - if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) { + if (rrdvar_flags(rv) & (RRDVAR_FLAG_CUSTOM_HOST_VAR | RRDVAR_FLAG_CUSTOM_CHART_VAR)) { if (!opts->host_header_printed) { opts->host_header_printed = 1; @@ -519,7 +519,6 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( PROMETHEUS_OUTPUT_OPTIONS output_options) { SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT); - rrdhost_rdlock(host); char hostname[PROMETHEUS_ELEMENT_MAX + 1]; prometheus_label_copy(hostname, rrdhost_hostname(host), PROMETHEUS_ELEMENT_MAX); @@ -564,17 +563,14 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( .host_header_printed = 0 }; - rrdvar_walkthrough_read(host->rrdvar_root_index, print_host_variables, &opts); + rrdvar_walkthrough_read(host->rrdvars, print_host_variables_callback, &opts); } // for each chart RRDSET *st; - rrdset_foreach_read(st, host) - { + rrdset_foreach_read(st, host) { if (likely(can_send_rrdset(instance, st, filter))) { - rrdset_rdlock(st); - char chart[PROMETHEUS_ELEMENT_MAX + 1]; char context[PROMETHEUS_ELEMENT_MAX + 1]; char family[PROMETHEUS_ELEMENT_MAX + 1]; @@ -615,8 +611,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( // for each dimension RRDDIM *rd; - rrddim_foreach_read(rd, st) - { + rrddim_foreach_read(rd, st) { if (rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { char dimension[PROMETHEUS_ELEMENT_MAX + 1]; char *suffix = ""; @@ -665,7 +660,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( buffer_sprintf(wb, "# TYPE %s_%s%s %s\n", prefix, context, suffix, p.type); generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector); - } else { + } + else { // the dimensions of the chart, do not have the same algorithm, multiplier or divisor // we create a metric per dimension @@ -683,7 +679,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector); } - } else { + } + else { // we need average or sum of the data time_t first_time = instance->after; @@ -750,12 +747,11 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( } } } - - rrdset_unlock(st); + rrddim_foreach_done(rd); } } + rrdset_foreach_done(st); - rrdhost_unlock(host); simple_pattern_free(filter); } diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c index ad337513eb..15f5e41036 100644 --- a/exporting/prometheus/remote_write/remote_write.c +++ b/exporting/prometheus/remote_write/remote_write.c @@ -321,12 +321,12 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * return 0; } -static int format_variable_prometheus_remote_write_callback(const char *name_txt __maybe_unused, void *rv_ptr, void *data) { - RRDVAR *rv = rv_ptr; +static int format_variable_prometheus_remote_write_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rv_ptr __maybe_unused, void *data) { + const RRDVAR_ACQUIRED *rv = (const RRDVAR_ACQUIRED *)item; struct prometheus_remote_write_variables_callback_options *opts = data; - if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) { + if (rrdvar_flags(rv) & (RRDVAR_FLAG_CUSTOM_HOST_VAR | RRDVAR_FLAG_CUSTOM_CHART_VAR)) { RRDHOST *host = opts->host; struct instance *instance = opts->instance; struct simple_connector_data *simple_connector_data = @@ -363,7 +363,7 @@ int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST .now = now_realtime_usec(), }; - return rrdvar_walkthrough_read(host->rrdvar_root_index, format_variable_prometheus_remote_write_callback, &opt); + return rrdvar_walkthrough_read(host->rrdvars, format_variable_prometheus_remote_write_callback, &opt); } /** diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c index 735925d478..f83a474f9b 100644 --- a/exporting/tests/netdata_doubles.c +++ b/exporting/tests/netdata_doubles.c @@ -242,14 +242,6 @@ void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle) function_called(); } -void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value) -{ - (void)chart_uuid; - (void)source_type; - (void)label; - (void)value; -} - void rrdcalc_update_rrdlabels(RRDSET *st) { (void)st; diff --git a/health/health.c b/health/health.c index c1045c3d81..be8b50ebb3 100644 --- a/health/health.c +++ b/health/health.c @@ -153,63 +153,42 @@ static void health_reload_host(RRDHOST *host) { char *stock_path = health_stock_config_dir(); // free all running alarms - rrdhost_wrlock(host); - - while(host->alarms_templates) - rrdcalctemplate_unlink_and_free(host, host->alarms_templates); - - while(host->host_alarms) - rrdcalc_unlink_and_free(host, host->host_alarms); - - RRDCALC *rc,*nc; - for(rc = host->alarms_with_foreach; rc ; rc = nc) { - nc = rc->next; - rrdcalc_free(rc); - } - host->alarms_with_foreach = NULL; - - rrdhost_unlock(host); + rrdcalc_delete_all(host); + rrdcalctemplate_delete_all(host); // invalidate all previous entries in the alarm log + netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); ALARM_ENTRY *t; for(t = host->health_log.alarms ; t ; t = t->next) { if(t->new_status != RRDCALC_STATUS_REMOVED) t->flags |= HEALTH_ENTRY_FLAG_UPDATED; } + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); - rrdhost_rdlock(host); // reset all thresholds to all charts RRDSET *st; rrdset_foreach_read(st, host) { st->green = NAN; st->red = NAN; } - rrdhost_unlock(host); + rrdset_foreach_done(st); // load the new alarms - rrdhost_wrlock(host); health_readdir(host, user_path, stock_path, NULL); //Discard alarms with labels that do not apply to host - rrdcalc_labels_unlink_alarm_from_host(host); + rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host); // link the loaded alarms to their charts - RRDDIM *rd; rrdset_foreach_write(st, host) { if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) continue; - rrdsetcalc_link_matching(st); - rrdcalctemplate_link_matching(st); - //This loop must be the last, because ` rrdcalctemplate_link_matching` will create alarms related to it. - rrdset_rdlock(st); - rrddim_foreach_read(rd, st) { - rrdcalc_link_to_rrddim(rd, st, host); - } - rrdset_unlock(st); + rrdcalc_link_matching_alerts_to_rrdset(st); + rrdcalctemplate_link_matching_templates_to_rrdset(st); } + rrdset_foreach_done(st); - rrdhost_unlock(host); } /** @@ -323,7 +302,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE); crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE); - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; @@ -351,6 +330,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { expr = rc->warning; } } + foreach_rrdcalc_in_rrdhost_done(rc); if (n_warn+n_crit>1) qsort (active_alerts, n_warn+n_crit, sizeof(active_alerts_t), compare_active_alerts); @@ -477,13 +457,13 @@ static inline void health_alarm_log_process(RRDHOST *host) { } } + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); + // remember this for the next iteration host->health_last_processed_id = first_waiting; bool cleanup_excess_log_entries = host->health_log.count > host->health_log.max; - netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); - if (!cleanup_excess_log_entries) return; @@ -554,10 +534,8 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) } int update_every = rc->rrdset->update_every; - rrdset_rdlock(rc->rrdset); - time_t first = rrdset_first_entry_t_nolock(rc->rrdset); - time_t last = rrdset_last_entry_t_nolock(rc->rrdset); - rrdset_unlock(rc->rrdset); + time_t first = rrdset_first_entry_t(rc->rrdset); + time_t last = rrdset_last_entry_t(rc->rrdset); if(unlikely(now + update_every < first /* || now - update_every > last */)) { debug(D_HEALTH @@ -653,29 +631,29 @@ static SILENCE_TYPE check_silenced(RRDCALC *rc, const char *host, SILENCERS *sil * @return It returns 1 case rrdcalc_flags is DISABLED or 0 otherwise */ static int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) { - uint32_t rrdcalc_flags_old = rc->rrdcalc_flags; + uint32_t rrdcalc_flags_old = rc->run_flags; // Clear the flags - rc->rrdcalc_flags &= ~(RRDCALC_FLAG_DISABLED | RRDCALC_FLAG_SILENCED); + rc->run_flags &= ~(RRDCALC_FLAG_DISABLED | RRDCALC_FLAG_SILENCED); if (unlikely(silencers->all_alarms)) { - if (silencers->stype == STYPE_DISABLE_ALARMS) rc->rrdcalc_flags |= RRDCALC_FLAG_DISABLED; - else if (silencers->stype == STYPE_SILENCE_NOTIFICATIONS) rc->rrdcalc_flags |= RRDCALC_FLAG_SILENCED; + if (silencers->stype == STYPE_DISABLE_ALARMS) rc->run_flags |= RRDCALC_FLAG_DISABLED; + else if (silencers->stype == STYPE_SILENCE_NOTIFICATIONS) rc->run_flags |= RRDCALC_FLAG_SILENCED; } else { SILENCE_TYPE st = check_silenced(rc, rrdhost_hostname(host), silencers); - if (st == STYPE_DISABLE_ALARMS) rc->rrdcalc_flags |= RRDCALC_FLAG_DISABLED; - else if (st == STYPE_SILENCE_NOTIFICATIONS) rc->rrdcalc_flags |= RRDCALC_FLAG_SILENCED; + if (st == STYPE_DISABLE_ALARMS) rc->run_flags |= RRDCALC_FLAG_DISABLED; + else if (st == STYPE_SILENCE_NOTIFICATIONS) rc->run_flags |= RRDCALC_FLAG_SILENCED; } - if (rrdcalc_flags_old != rc->rrdcalc_flags) { + if (rrdcalc_flags_old != rc->run_flags) { info("Alarm silencing changed for host '%s' alarm '%s': Disabled %s->%s Silenced %s->%s", rrdhost_hostname(host), rrdcalc_name(rc), (rrdcalc_flags_old & RRDCALC_FLAG_DISABLED)?"true":"false", - (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false", + (rc->run_flags & RRDCALC_FLAG_DISABLED)?"true":"false", (rrdcalc_flags_old & RRDCALC_FLAG_SILENCED)?"true":"false", - (rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)?"true":"false" + (rc->run_flags & RRDCALC_FLAG_SILENCED)?"true":"false" ); } - if (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED) + if (rc->run_flags & RRDCALC_FLAG_DISABLED) return 1; else return 0; @@ -683,36 +661,38 @@ static int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) { // Create alarms for dimensions that have been added to charts // since the previous iteration. -static void init_pending_foreach_alarms(RRDHOST *host) { +static void health_execute_pending_updates(RRDHOST *host) { RRDSET *st; - RRDDIM *rd; if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS)) return; - rrdhost_wrlock(host); - - rrdset_foreach_write(st, host) { - if (!rrdset_flag_check(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS)) + rrdset_foreach_reentrant(st, host) { + if(!rrdset_flag_check(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS)) continue; - rrdset_rdlock(st); - + RRDDIM *rd; rrddim_foreach_read(rd, st) { - if (!rrddim_flag_check(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM)) + if(!rrddim_flag_check(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARMS)) continue; - rrdcalc_link_to_rrddim(rd, st, host); + RRDCALCTEMPLATE *rt; + foreach_rrdcalctemplate_read(host, rt) { + if(!rt->foreach_dimension_pattern) + continue; - rrddim_flag_clear(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM); + if(rrdcalctemplate_check_rrdset_conditions(rt, st, host)) + rrdcalctemplate_check_rrddim_conditions_and_link(rt, st, rd, host); + } + foreach_rrdcalctemplate_done(rt); + + rrddim_flag_clear(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARMS); } - + rrddim_foreach_done(rd); rrdset_flag_clear(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS); - rrdset_unlock(st); } - + rrdset_foreach_done(st); rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS); - rrdhost_unlock(host); } /** @@ -759,7 +739,7 @@ void *health_main(void *ptr) { time_t now = now_realtime_sec(); time_t hibernation_delay = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60); - rrdcalc_labels_unlink(); + rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts(); unsigned int loop = 0; #ifdef ENABLE_ACLK @@ -829,13 +809,14 @@ void *health_main(void *ptr) { if(likely(!host->health_log_fp) && (loop == 1 || loop % cleanup_sql_every_loop == 0)) sql_health_alarm_log_cleanup(host); - init_pending_foreach_alarms(host); + health_execute_pending_updates(host); worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK); - rrdhost_rdlock(host); // the first loop is to lookup values from the db - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { + + rrdcalc_update_info_using_rrdset_labels(rc); if (update_disabled_silenced(host, rc)) continue; @@ -876,7 +857,7 @@ void *health_main(void *ptr) { rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0); if (ae) { - health_alarm_log(host, ae); + health_alarm_log_add_entry(host, ae); rc->old_status = rc->status; rc->status = RRDCALC_STATUS_REMOVED; rc->last_status_change = now; @@ -891,14 +872,14 @@ void *health_main(void *ptr) { } if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) { - if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE)) - rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUNNABLE; + if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE)) + rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE; continue; } runnable++; rc->old_value = rc->value; - rc->rrdcalc_flags |= RRDCALC_FLAG_RUNNABLE; + rc->run_flags |= RRDCALC_FLAG_RUNNABLE; // ------------------------------------------------------------ // if there is database lookup, do it @@ -919,13 +900,13 @@ void *health_main(void *ptr) { if (unlikely(ret != 200)) { // database lookup failed rc->value = NAN; - rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR; + rc->run_flags |= RRDCALC_FLAG_DB_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret ); } else - rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR; + rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR; /* - RRDCALC_FLAG_DB_STALE not currently used if (unlikely(old_db_timestamp == rc->db_before)) { @@ -945,14 +926,14 @@ void *health_main(void *ptr) { if (unlikely(value_is_null)) { // collected value is null rc->value = NAN; - rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN; + rc->run_flags |= RRDCALC_FLAG_DB_NAN; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc) ); } else - rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN; + rc->run_flags &= ~RRDCALC_FLAG_DB_NAN; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT, rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value @@ -968,14 +949,14 @@ void *health_main(void *ptr) { if (unlikely(!expression_evaluate(rc->calculation))) { // calculation failed rc->value = NAN; - rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR; + rc->run_flags |= RRDCALC_FLAG_CALC_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg) ); } else { - rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR; + rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value " NETDATA_DOUBLE_FORMAT @@ -985,25 +966,17 @@ void *health_main(void *ptr) { ); rc->value = rc->calculation->result; - - if (rc->local) rc->local->last_updated = now; - if (rc->family) rc->family->last_updated = now; - if (rc->hostid) rc->hostid->last_updated = now; - if (rc->hostname) rc->hostname->last_updated = now; } } } - - rrdhost_unlock(host); + foreach_rrdcalc_in_rrdhost_done(rc); if (unlikely(runnable && !netdata_exit)) { - rrdhost_rdlock(host); - - foreach_rrdcalc_in_rrdhost(host, rc) { - if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))) + foreach_rrdcalc_in_rrdhost_read(host, rc) { + if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE))) continue; - if (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED) { + if (rc->run_flags & RRDCALC_FLAG_DISABLED) { continue; } RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED; @@ -1017,7 +990,7 @@ void *health_main(void *ptr) { if (unlikely(!expression_evaluate(rc->warning))) { // calculation failed - rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR; + rc->run_flags |= RRDCALC_FLAG_WARN_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s", @@ -1025,7 +998,7 @@ void *health_main(void *ptr) { buffer_tostring(rc->warning->error_msg) ); } else { - rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR; + rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value " NETDATA_DOUBLE_FORMAT ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), @@ -1043,7 +1016,7 @@ void *health_main(void *ptr) { if (unlikely(!expression_evaluate(rc->critical))) { // calculation failed - rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR; + rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s", @@ -1051,7 +1024,7 @@ void *health_main(void *ptr) { buffer_tostring(rc->critical->error_msg) ); } else { - rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR; + rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR; debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value " NETDATA_DOUBLE_FORMAT ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), @@ -1156,13 +1129,13 @@ void *health_main(void *ptr) { rc->info, rc->delay_last, ( - ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | - ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) | + ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | + ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) | (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0) ) ); - health_alarm_log(host, ae); + health_alarm_log_add_entry(host, ae); rc->last_status_change = now; rc->old_status = rc->status; @@ -1175,20 +1148,20 @@ void *health_main(void *ptr) { if (next_run > rc->next_update) next_run = rc->next_update; } + foreach_rrdcalc_in_rrdhost_done(rc); // process repeating alarms - RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { int repeat_every = 0; if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) { if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) { - rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUN_ONCE; + rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE; repeat_every = rc->warn_repeat_every; } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) { - rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUN_ONCE; + rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE; repeat_every = rc->crit_repeat_every; } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) { - if(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUN_ONCE)) { + if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE)) { if(rc->old_status == RRDCALC_STATUS_CRITICAL) { repeat_every = 1; } else if (rc->old_status == RRDCALC_STATUS_WARNING) { @@ -1230,25 +1203,24 @@ void *health_main(void *ptr) { rc->info, rc->delay_last, ( - ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | - ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) | + ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | + ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) | (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0) ) ); ae->last_repeat = rc->last_repeat; - if (!(rc->rrdcalc_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) { + if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) { ae->flags |= HEALTH_ENTRY_RUN_ONCE; } - rc->rrdcalc_flags |= RRDCALC_FLAG_RUN_ONCE; + rc->run_flags |= RRDCALC_FLAG_RUN_ONCE; health_process_notifications(host, ae); debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id); health_alarm_wait_for_execution(ae); health_alarm_log_free_one_nochecks_nounlink(ae); } } - - rrdhost_unlock(host); + foreach_rrdcalc_in_rrdhost_done(rc); } if (unlikely(netdata_exit)) diff --git a/health/health.h b/health/health.h index aae608b522..aa7a5c65ed 100644 --- a/health/health.h +++ b/health/health.h @@ -74,7 +74,7 @@ extern ALARM_ENTRY* health_create_alarm_entry( int delay, uint32_t flags); -extern void health_alarm_log(RRDHOST *host, ALARM_ENTRY *ae); +extern void health_alarm_log_add_entry(RRDHOST *host, ALARM_ENTRY *ae); extern void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath); extern char *health_user_config_dir(void); @@ -90,6 +90,4 @@ extern void health_label_log_save(RRDHOST *host); extern char *health_edit_command_from_source(const char *source); extern void sql_refresh_hashes(void); -extern SIMPLE_PATTERN *health_pattern_from_foreach(const char *s); - #endif //NETDATA_HEALTH_H diff --git a/health/health_config.c b/health/health_config.c index eb09a5f81d..365c4d015d 100644 --- a/health/health_config.c +++ b/health/health_config.c @@ -33,122 +33,6 @@ #define HEALTH_HOST_LABEL_KEY "host labels" #define HEALTH_FOREACH_KEY "foreach" -static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) { - if(!rc->chart) { - error("Health configuration for alarm '%s' does not have a chart", rrdcalc_name(rc)); - return 0; - } - - if(!rc->update_every) { - error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rrdcalc_chart_name(rc), rrdcalc_name(rc)); - return 0; - } - - if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->calculation && !rc->warning && !rc->critical) { - error("Health configuration for alarm '%s.%s' is useless (no db lookup, no calculation, no warning and no critical expressions)", rrdcalc_chart_name(rc), rrdcalc_name(rc)); - return 0; - } - - if (rrdcalc_exists(host, rrdcalc_chart_name(rc), rrdcalc_name(rc))) - return 0; - - rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id); - - debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO - ", red " NETDATA_DOUBLE_FORMAT_AUTO - ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", - rrdcalc_chart_name(rc), - rrdcalc_name(rc), - rc->id, - (rc->exec)?rrdcalc_exec(rc):"DEFAULT", - (rc->recipient)?rrdcalc_recipient(rc):"DEFAULT", - rc->green, - rc->red, - (int)rc->group, - rc->after, - rc->before, - rc->options, - (rc->dimensions)?rrdcalc_dimensions(rc):"NONE", - (rc->foreachdim)?rrdcalc_foreachdim(rc):"NONE", - rc->update_every, - (rc->calculation)?rc->calculation->parsed_as:"NONE", - (rc->warning)?rc->warning->parsed_as:"NONE", - (rc->critical)?rc->critical->parsed_as:"NONE", - rrdcalc_source(rc), - rc->delay_up_duration, - rc->delay_down_duration, - rc->delay_max_duration, - rc->delay_multiplier, - rc->warn_repeat_every, - rc->crit_repeat_every - ); - - rrdcalc_add_to_host(host, rc); - - return 1; -} - -static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) { - if(unlikely(!rt->context)) { - error("Health configuration for template '%s' does not have a context", rrdcalctemplate_name(rt)); - return 0; - } - - if(unlikely(!rt->update_every)) { - error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rrdcalctemplate_name(rt)); - return 0; - } - - if(unlikely(!RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) && !rt->calculation && !rt->warning && !rt->critical)) { - error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rrdcalctemplate_name(rt)); - return 0; - } - - RRDCALCTEMPLATE *t; - foreach_rrdcalctemplate_in_rrdhost(host, t) { - if(unlikely(t->name == rt->name && !strcmp(t->family_match?rrdcalctemplate_family_match(t):"*", rt->family_match?rrdcalctemplate_family_match(rt):"*"))) { - info("Health configuration template '%s' already exists for host '%s'.", rrdcalctemplate_name(rt), rrdhost_hostname(host)); - return 0; - } - } - - if(rt->foreachdim) - DOUBLE_LINKED_LIST_PREPEND_UNSAFE(host->alarms_templates, rt, prev, next); - else - DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->alarms_templates, rt, prev, next); - - debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO - ", red " NETDATA_DOUBLE_FORMAT_AUTO - ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", - rrdcalctemplate_name(rt), - (rt->context)?string2str(rt->context):"NONE", - (rt->exec)?rrdcalctemplate_exec(rt):"DEFAULT", - (rt->recipient)?rrdcalctemplate_recipient(rt):"DEFAULT", - rt->green, - rt->red, - (int)rt->group, - rt->after, - rt->before, - rt->options, - (rt->dimensions)?rrdcalctemplate_dimensions(rt):"NONE", - (rt->foreachdim)?rrdcalctemplate_foreachdim(rt):"NONE", - rt->update_every, - (rt->calculation)?rt->calculation->parsed_as:"NONE", - (rt->warning)?rt->warning->parsed_as:"NONE", - (rt->critical)?rt->critical->parsed_as:"NONE", - rrdcalctemplate_source(rt), - rt->delay_up_duration, - rt->delay_down_duration, - rt->delay_max_duration, - rt->delay_multiplier, - rt->warn_repeat_every, - rt->crit_repeat_every - ); - - - return 1; -} - static inline int health_parse_delay( size_t line, const char *filename, char *string, int *delay_up_duration, @@ -249,7 +133,7 @@ static inline uint32_t health_parse_options(const char *s) { buf[count] = '\0'; if(!strcasecmp(buf, "no-clear-notification") || !strcasecmp(buf, "no-clear")) - options |= RRDCALC_FLAG_NO_CLEAR_NOTIFICATION; + options |= RRDCALC_OPTION_NO_CLEAR_NOTIFICATION; else error("Ignoring unknown alarm option '%s'", buf); } @@ -308,13 +192,21 @@ static inline int health_parse_repeat( * * @param s the string that will be used to create the simple pattern. */ -SIMPLE_PATTERN *health_pattern_from_foreach(const char *s) { + +static void dimension_remove_pipe_comma(char *str) { + while(*str) { + if(*str == '|' || *str == ',') *str = ' '; + str++; + } +} + +static SIMPLE_PATTERN *health_pattern_from_foreach(const char *s) { char *convert= strdupz(s); SIMPLE_PATTERN *val = NULL; + if(convert) { dimension_remove_pipe_comma(convert); val = simple_pattern_create(convert, NULL, SIMPLE_PATTERN_EXACT); - freez(convert); } @@ -324,7 +216,7 @@ SIMPLE_PATTERN *health_pattern_from_foreach(const char *s) { static inline int health_parse_db_lookup( size_t line, const char *filename, char *string, RRDR_GROUPING *group_method, int *after, int *before, int *every, - uint32_t *options, STRING **dimensions, STRING **foreachdim + RRDCALC_OPTIONS *options, STRING **dimensions, STRING **foreachdim ) { debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s: %s", line, filename, string); @@ -335,7 +227,7 @@ static inline int health_parse_db_lookup( *after = 0; *before = 0; *every = 0; - *options = 0; + *options = (*options) & RRDCALC_ALL_OPTIONS_EXCLUDING_THE_RRDR_ONES; // preserve rrdcalc options char *s = string, *key; @@ -644,16 +536,20 @@ static int health_readfile(const char *filename, void *data) { if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) { if(rc) { - if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) { - rrdcalc_free(rc); - } + if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this) + rrdcalc_free_unused_rrdcalc_loaded_from_config(rc); + else + rrdcalc_add_from_config(host, rc); + // health_add_alarms_loop(host, rc, ignore_this) ; } if(rt) { - if (!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) { - rrdcalctemplate_free(rt); - } + if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this) + rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(rt); + else + rrdcalctemplate_add_from_config(host, rt); + rt = NULL; } @@ -688,15 +584,19 @@ static int health_readfile(const char *filename, void *data) { else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) { if(rc) { // health_add_alarms_loop(host, rc, ignore_this) ; - if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) - rrdcalc_free(rc); + if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this) + rrdcalc_free_unused_rrdcalc_loaded_from_config(rc); + else + rrdcalc_add_from_config(host, rc); rc = NULL; } if(rt) { - if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) - rrdcalctemplate_free(rt); + if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this) + rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(rt); + else + rrdcalctemplate_add_from_config(host, rt); } rt = callocz(1, sizeof(RRDCALCTEMPLATE)); @@ -811,10 +711,10 @@ static int health_readfile(const char *filename, void *data) { else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) { alert_cfg->lookup = string_strdupz(value); health_parse_db_lookup(line, filename, value, &rc->group, &rc->after, &rc->before, - &rc->update_every, &rc->options, &rc->dimensions, &rc->foreachdim); + &rc->update_every, &rc->options, &rc->dimensions, &rc->foreach_dimension); - if(rc->foreachdim) - rc->spdim = health_pattern_from_foreach(rrdcalc_foreachdim(rc)); + if(rc->foreach_dimension) + rc->foreach_dimension_pattern = health_pattern_from_foreach(rrdcalc_foreachdim(rc)); if (rc->after) { if (rc->dimensions) @@ -1071,10 +971,10 @@ static int health_readfile(const char *filename, void *data) { else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) { alert_cfg->lookup = string_strdupz(value); health_parse_db_lookup(line, filename, value, &rt->group, &rt->after, &rt->before, - &rt->update_every, &rt->options, &rt->dimensions, &rt->foreachdim); + &rt->update_every, &rt->options, &rt->dimensions, &rt->foreach_dimension); - if(rt->foreachdim) - rt->spdim = health_pattern_from_foreach(rrdcalctemplate_foreachdim(rt)); + if(rt->foreach_dimension) + rt->foreach_dimension_pattern = health_pattern_from_foreach(rrdcalctemplate_foreachdim(rt)); if (rt->after) { if (rt->dimensions) @@ -1237,15 +1137,17 @@ static int health_readfile(const char *filename, void *data) { if(rc) { //health_add_alarms_loop(host, rc, ignore_this) ; - if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) { - rrdcalc_free(rc); - } + if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this) + rrdcalc_free_unused_rrdcalc_loaded_from_config(rc); + else + rrdcalc_add_from_config(host, rc); } if(rt) { - if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) { - rrdcalctemplate_free(rt); - } + if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this) + rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(rt); + else + rrdcalctemplate_add_from_config(host, rt); } if (alert_cfg) diff --git a/health/health_json.c b/health/health_json.c index a19526d344..793c85c46f 100644 --- a/health/health_json.c +++ b/health/health_json.c @@ -116,7 +116,6 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) } void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart) { - netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); buffer_strcat(wb, "["); @@ -125,6 +124,8 @@ void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *char STRING *chart_string = string_strdupz(chart); + netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); + ALARM_ENTRY *ae; for (ae = host->health_log.alarms; ae && count < max; ae = ae->next) { if ((ae->unique_id > after) && (!chart || chart_string == ae->chart)) { @@ -135,11 +136,11 @@ void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *char } } + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); + string_freez(chart_string); buffer_strcat(wb, "\n]\n"); - - netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); } static inline void health_rrdcalc_values2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC *rc) { @@ -216,8 +217,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC , rc->component?rrdcalc_component(rc):"Unknown" , rc->type?rrdcalc_type(rc):"Unknown" , (rc->rrdset)?"true":"false" - , (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false" - , (rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)?"true":"false" + , (rc->run_flags & RRDCALC_FLAG_DISABLED)?"true":"false" + , (rc->run_flags & RRDCALC_FLAG_SILENCED)?"true":"false" , rc->exec?rrdcalc_exec(rc):string2str(host->health_default_exec) , rc->recipient?rrdcalc_recipient(rc):string2str(host->health_default_recipient) , rrdcalc_source(rc) @@ -241,7 +242,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC , (unsigned long)rc->times_repeat ); - if(unlikely(rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)) { + if(unlikely(rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)) { buffer_strcat(wb, "\t\t\t\"no_clear_notification\": true,\n"); } @@ -306,8 +307,6 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL char *tok = NULL; char *p = NULL; - rrdhost_rdlock(host); - if (contexts) { p = (char*)buffer_tostring(contexts); while(p && *p && (tok = mystrsep(&p, ", |"))) { @@ -315,7 +314,7 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL STRING *tok_string = string_strdupz(tok); - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; if (unlikely(!rrdset_is_available_for_exporting_and_alarms(rc->rrdset))) @@ -325,12 +324,13 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL && ((status==RRDCALC_STATUS_RAISED)?(rc->status >= RRDCALC_STATUS_WARNING):rc->status == status))) numberOfAlarms++; } + foreach_rrdcalc_in_rrdhost_done(rc); string_freez(tok_string); } } else { - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; if (unlikely(!rrdset_is_available_for_exporting_and_alarms(rc->rrdset))) @@ -338,16 +338,16 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL if(unlikely((status==RRDCALC_STATUS_RAISED)?(rc->status >= RRDCALC_STATUS_WARNING):rc->status == status)) numberOfAlarms++; } + foreach_rrdcalc_in_rrdhost_done(rc); } buffer_sprintf(wb, "%d", numberOfAlarms); - rrdhost_unlock(host); } static void health_alarms2json_fill_alarms(RRDHOST *host, BUFFER *wb, int all, void (*fp)(RRDHOST *, BUFFER *, RRDCALC *)) { RRDCALC *rc; int i = 0; - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; @@ -361,10 +361,10 @@ static void health_alarms2json_fill_alarms(RRDHOST *host, BUFFER *wb, int all, v fp(host, wb, rc); i++; } + foreach_rrdcalc_in_rrdhost_done(rc); } void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) { - rrdhost_rdlock(host); buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\"," "\n\t\"latest_alarm_log_unique_id\": %u," "\n\t\"status\": %s," @@ -377,17 +377,17 @@ void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) { health_alarms2json_fill_alarms(host, wb, all, health_rrdcalc2json_nolock); +// rrdhost_rdlock(host); // buffer_strcat(wb, "\n\t},\n\t\"templates\": {"); // RRDCALCTEMPLATE *rt; // for(rt = host->templates; rt ; rt = rt->next) // health_rrdcalctemplate2json_nolock(wb, rt); +// rrdhost_unlock(host); buffer_strcat(wb, "\n\t}\n}\n"); - rrdhost_unlock(host); } void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all) { - rrdhost_rdlock(host); buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\"," "\n\t\"alarms\": {\n", rrdhost_hostname(host)); @@ -395,7 +395,6 @@ void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all) { health_alarms2json_fill_alarms(host, wb, all, health_rrdcalc_values2json_nolock); buffer_strcat(wb, "\n\t}\n}\n"); - rrdhost_unlock(host); } static int have_recent_alarm(RRDHOST *host, uint32_t alarm_id, time_t mark) diff --git a/health/health_log.c b/health/health_log.c index e1a8b66f42..8105e01aea 100644 --- a/health/health_log.c +++ b/health/health_log.c @@ -182,12 +182,15 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char size_t line = 0, len = 0; ssize_t loaded = 0, updated = 0, errored = 0, duplicate = 0; - netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); - - DICTIONARY *all_rrdcalcs = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + DICTIONARY *all_rrdcalcs = dictionary_create( + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) + foreach_rrdcalc_in_rrdhost_read(host, rc) { dictionary_set(all_rrdcalcs, rrdcalc_name(rc), rc, sizeof(*rc)); + } + foreach_rrdcalc_in_rrdhost_done(rc); + + netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); while((s = fgets_trim_len(buf, 65536, fp, &len))) { host->health_log_entries_written++; @@ -394,11 +397,11 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char } } + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); + dictionary_destroy(all_rrdcalcs); all_rrdcalcs = NULL; - netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); - freez(buf); if(!host->health_max_unique_id) host->health_max_unique_id = (uint32_t)now_realtime_sec(); @@ -510,7 +513,7 @@ inline ALARM_ENTRY* health_create_alarm_entry( return ae; } -inline void health_alarm_log( +inline void health_alarm_log_add_entry( RRDHOST *host, ALARM_ENTRY *ae ) { @@ -568,8 +571,6 @@ inline void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae) { } inline void health_alarm_log_free(RRDHOST *host) { - rrdhost_check_wrlock(host); - netdata_rwlock_wrlock(&host->health_log.alarm_log_rwlock); ALARM_ENTRY *ae; diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am index 5962323e87..1208d16c21 100644 --- a/libnetdata/Makefile.am +++ b/libnetdata/Makefile.am @@ -25,6 +25,7 @@ SUBDIRS = \ socket \ statistical \ storage_number \ + string \ threads \ url \ worker_utilization \ diff --git a/libnetdata/dictionary/README.md b/libnetdata/dictionary/README.md index 879c1bcc1c..bcd78ea262 100644 --- a/libnetdata/dictionary/README.md +++ b/libnetdata/dictionary/README.md @@ -18,7 +18,7 @@ Dictionaries provide an interface to: - **Delete** an item from the dictionary (provided its `name`) - **Traverse** the list of items in the dictionary -Dictionaries are **ordered**, meaning that the order they have been added is preserved while traversing them. The caller may reverse this order by passing the flag `DICTIONARY_FLAG_ADD_IN_FRONT` when creating the dictionary. +Dictionaries are **ordered**, meaning that the order they have been added is preserved while traversing them. The caller may reverse this order by passing the flag `DICT_OPTION_ADD_IN_FRONT` when creating the dictionary. Dictionaries guarantee **uniqueness** of all items added to them, meaning that only one item with a given name can exist in the dictionary at any given time. @@ -35,21 +35,21 @@ In **clone** mode, the dictionary guarantees that all operations on the dictiona 1.`dictionary_register_insert_callback()` that will be called just after the insertion of an item to the dictionary, or after the replacement of the value of a dictionary item (but while the dictionary is write-locked - if locking is enabled). 2. `dictionary_register_delete_callback()` that will be called just prior to the deletion of an item from the dictionary, or prior to the replacement of the value of a dictionary item (but while the dictionary is write-locked - if locking is enabled). - 3. `dictionary_register_conflict_callback()` that will be called when `DICTIONARY_FLAG_DONT_OVERWRITE_VALUE` is set and another value is attempted to be inserted for the same key. + 3. `dictionary_register_conflict_callback()` that will be called when `DICT_OPTION_DONT_OVERWRITE_VALUE` is set and another value is attempted to be inserted for the same key. In **link** mode, the name and/or the value are just linked to the dictionary item, and it is the user's responsibility to free the memory they use after an item is deleted from the dictionary or when the dictionary is destroyed. By default, **clone** mode is used for both the name and the value. -To use **link** mode for names, add `DICTIONARY_FLAG_NAME_LINK_DONT_CLONE` to the flags when creating the dictionary. +To use **link** mode for names, add `DICT_OPTION_NAME_LINK_DONT_CLONE` to the flags when creating the dictionary. -To use **link** mode for values, add `DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE` to the flags when creating the dictionary. +To use **link** mode for values, add `DICT_OPTION_VALUE_LINK_DONT_CLONE` to the flags when creating the dictionary. ## Locks The dictionary allows both **single-threaded** operation (no locks - faster) and **multi-threaded** operation utilizing a read-write lock. -The default is **multi-threaded**. To enable **single-threaded** add `DICTIONARY_FLAG_SINGLE_THREADED` to the flags when creating the dictionary. +The default is **multi-threaded**. To enable **single-threaded** add `DICT_OPTION_SINGLE_THREADED` to the flags when creating the dictionary. ## Hash table operations @@ -72,7 +72,7 @@ This call is used to: - **add** an item to the dictionary. - **reset** the value of an existing item in the dictionary. -If **resetting** is not desired, add `DICTIONARY_FLAG_DONT_OVERWRITE_VALUE` to the flags when creating the dictionary. In this case, `dictionary_set()` will return the value of the original item found in the dictionary instead of resetting it and the value passed to the call will be ignored. Optionally a conflict callback function can be registered, to manipulate (probably merge or extend) the original value, based on the new value attempted to be added to the dictionary. +If **resetting** is not desired, add `DICT_OPTION_DONT_OVERWRITE_VALUE` to the flags when creating the dictionary. In this case, `dictionary_set()` will return the value of the original item found in the dictionary instead of resetting it and the value passed to the call will be ignored. Optionally a conflict callback function can be registered, to manipulate (probably merge or extend) the original value, based on the new value attempted to be added to the dictionary. For **multi-threaded** operation, the `dictionary_set()` calls get an exclusive write lock on the dictionary. @@ -143,7 +143,7 @@ Example: ```c // create the dictionary -DICTIONARY *dict = dictionary_create(DICTIONARY_FLAGS_NONE); +DICTIONARY *dict = dictionary_create(DICT_OPTION_NONE); // add an item to it dictionary_set(dict, "name", "value", 6); @@ -189,7 +189,7 @@ There are 4 calls: - `dictionary_walkthrough_read()` and `dictionary_sorted_walkthrough_read()` that acquire a shared read lock, and they call a callback function for every item of the dictionary. The callback function may use the unsafe versions of the `dictionary_get()` calls to lookup other items in the dictionary, but it should not attempt to add or remove items to/from the dictionary. - `dictionary_walkthrough_write()` and `dictionary_sorted_walkthrough_write()` that acquire an exclusive write lock, and they call a callback function for every item of the dictionary. This is to be used when items need to be added to or removed from the dictionary. The `write` versions can be used to delete any or all the items from the dictionary, including the currently working one. For the `sorted` version, all items in the dictionary maintain a reference counter, so all deletions are deferred until the sorted walkthrough finishes.** -The non sorted versions traverse the items in the same order they have been added to the dictionary (or the reverse order if the flag `DICTIONARY_FLAG_ADD_IN_FRONT` is set during dictionary creation). The sorted versions sort alphabetically the items based on their name, and then they traverse them in the sorted order. +The non sorted versions traverse the items in the same order they have been added to the dictionary (or the reverse order if the flag `DICT_OPTION_ADD_IN_FRONT` is set during dictionary creation). The sorted versions sort alphabetically the items based on their name, and then they traverse them in the sorted order. The callback function returns an `int`. If this value is negative, traversal of the dictionary is stopped immediately and the negative value is returned to the caller. If the returned value of all callback calls is zero or positive, the walkthrough functions return the sum of the return values of all callbacks. So, if you are just interested to know how many items fall into some condition, write a callback function that returns 1 when the item satisfies that condition and 0 when it does not and the walkthrough function will return how many tested positive. @@ -241,5 +241,5 @@ Since the dictionary uses a hash table and a double linked list, if the contract This is currently used in statsd: -- the data collection thread uses only `get` and `set`. It never uses `del`. New items are added at the front of the linked list (`DICTIONARY_FLAG_ADD_IN_FRONT`). +- the data collection thread uses only `get` and `set`. It never uses `del`. New items are added at the front of the linked list (`DICT_OPTION_ADD_IN_FRONT`). - the flushing thread is only traversing the dictionary up to the point it last traversed it (it uses a flag for that to know where it stopped last time). It never uses `get`, `set` or `del`. diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c index 259462e8b5..bec84f87a4 100644 --- a/libnetdata/dictionary/dictionary.c +++ b/libnetdata/dictionary/dictionary.c @@ -1,49 +1,132 @@ // SPDX-License-Identifier: GPL-3.0-or-later -// NOT TO BE USED BY USERS -#define DICTIONARY_FLAG_EXCLUSIVE_ACCESS (1 << 28) // there is only one thread accessing the dictionary -#define DICTIONARY_FLAG_DESTROYED (1 << 29) // this dictionary has been destroyed -#define DICTIONARY_FLAG_DEFER_ALL_DELETIONS (1 << 30) // defer all deletions of items in the dictionary - -// our reserved flags that cannot be set by users -#define DICTIONARY_FLAGS_RESERVED (DICTIONARY_FLAG_EXCLUSIVE_ACCESS|DICTIONARY_FLAG_DESTROYED|DICTIONARY_FLAG_DEFER_ALL_DELETIONS) - #define DICTIONARY_INTERNALS #include "../libnetdata.h" #include -typedef enum name_value_flags { - NAME_VALUE_FLAG_NONE = 0, - NAME_VALUE_FLAG_NAME_IS_ALLOCATED = (1 << 0), // the name pointer is a STRING - NAME_VALUE_FLAG_DELETED = (1 << 1), // this item is deleted, so it is not available for traversal - NAME_VALUE_FLAG_NEW_OR_UPDATED = (1 << 2), // this item is new or just updated (used by the react callback) +// runtime flags of the dictionary - must be checked with atomics +typedef enum { + DICT_FLAG_NONE = 0, + DICT_FLAG_DESTROYED = (1 << 0), // this dictionary has been destroyed +} DICT_FLAGS; + +#define dict_flag_check(dict, flag) (__atomic_load_n(&((dict)->flags), __ATOMIC_SEQ_CST) & (flag)) +#define dict_flag_set(dict, flag) __atomic_or_fetch(&((dict)->flags), flag, __ATOMIC_SEQ_CST) +#define dict_flag_clear(dict, flag) __atomic_and_fetch(&((dict)->flags), ~(flag), __ATOMIC_SEQ_CST) + +// flags macros +#define is_dictionary_destroyed(dict) dict_flag_check(dict, DICT_FLAG_DESTROYED) + +// configuration options macros +#define is_dictionary_single_threaded(dict) ((dict)->options & DICT_OPTION_SINGLE_THREADED) +#define is_view_dictionary(dict) ((dict)->master) +#define is_master_dictionary(dict) (!is_view_dictionary(dict)) + +typedef enum item_options { + ITEM_OPTION_NONE = 0, + ITEM_OPTION_ALLOCATED_NAME = (1 << 0), // the name pointer is a STRING + + // IMPORTANT: This is 1-bit - to add more change ITEM_OPTIONS_BITS +} ITEM_OPTIONS; + +typedef enum item_flags { + ITEM_FLAG_NONE = 0, + ITEM_FLAG_DELETED = (1 << 0), // this item is deleted, so it is not available for traversal + + // IMPORTANT: This is 8-bit +} ITEM_FLAGS; + +#define item_flag_check(item, flag) (__atomic_load_n(&((item)->flags), __ATOMIC_SEQ_CST) & (flag)) +#define item_flag_set(item, flag) __atomic_or_fetch(&((item)->flags), flag, __ATOMIC_SEQ_CST) +#define item_flag_clear(item, flag) __atomic_and_fetch(&((item)->flags), ~(flag), __ATOMIC_SEQ_CST) + +#define item_shared_flag_check(item, flag) (__atomic_load_n(&((item)->shared->flags), __ATOMIC_SEQ_CST) & (flag)) +#define item_shared_flag_set(item, flag) __atomic_or_fetch(&((item)->shared->flags), flag, __ATOMIC_SEQ_CST) +#define item_shared_flag_clear(item, flag) __atomic_and_fetch(&((item)->shared->flags), ~(flag), __ATOMIC_SEQ_CST) + +#define REFCOUNT_DELETING (-100) + +#define ITEM_FLAGS_TYPE uint8_t +#define KEY_LEN_TYPE uint32_t +#define VALUE_LEN_TYPE uint32_t + +#define ITEM_OPTIONS_BITS 1 +#define KEY_LEN_BITS ((sizeof(KEY_LEN_TYPE) * 8) - (sizeof(ITEM_FLAGS_TYPE) * 8) - ITEM_OPTIONS_BITS) +#define KEY_LEN_MAX ((1 << KEY_LEN_BITS) - 1) + +#define VALUE_LEN_BITS ((sizeof(VALUE_LEN_TYPE) * 8) - (sizeof(ITEM_FLAGS_TYPE) * 8)) +#define VALUE_LEN_MAX ((1 << VALUE_LEN_BITS) - 1) - // IMPORTANT: IF YOU ADD ANOTHER FLAG, YOU NEED TO ALLOCATE ANOTHER BIT TO FLAGS IN NAME_VALUE !!! -} NAME_VALUE_FLAGS; /* * Every item in the dictionary has the following structure. */ -typedef struct name_value { +typedef int32_t REFCOUNT; + +typedef struct dictionary_item_shared { + void *value; // the value of the dictionary item + + // the order of the following items is important! + // The total of their storage should be 64-bits + + REFCOUNT links; // how many links this item has + VALUE_LEN_TYPE value_len:VALUE_LEN_BITS; // the size of the value + ITEM_FLAGS_TYPE flags; // shared flags +} DICTIONARY_ITEM_SHARED; + +struct dictionary_item { #ifdef NETDATA_INTERNAL_CHECKS DICTIONARY *dict; #endif - struct name_value *next; // a double linked list to allow fast insertions and deletions - struct name_value *prev; + DICTIONARY_ITEM_SHARED *shared; - uint32_t refcount; // the reference counter - uint32_t value_len:29; // the size of the value (assumed binary) - uint8_t flags:3; // the flags for this item + struct dictionary_item *next; // a double linked list to allow fast insertions and deletions + struct dictionary_item *prev; - void *value; // the value of the dictionary item union { - STRING *string_name; // the name of the dictionary item - char *caller_name; // the user supplied string pointer + STRING *string_name; // the name of the dictionary item + char *caller_name; // the user supplied string pointer +// void *key_ptr; // binary key pointer }; -} NAME_VALUE; + + // the order of the following items is important! + // The total of their storage should be 64-bits + + REFCOUNT refcount; // the private reference counter + + KEY_LEN_TYPE key_len:KEY_LEN_BITS; // the size of key indexed (for strings, including the null terminator) + // this is (2^23 - 1) = 8.388.607 bytes max key length. + + ITEM_OPTIONS options:ITEM_OPTIONS_BITS; // permanent configuration options + // (no atomic operations on this - they never change) + + ITEM_FLAGS_TYPE flags; // runtime changing flags for this item (atomic operations on this) + // cannot be a bit field because of atomics. +}; + +struct dictionary_hooks { + REFCOUNT links; + usec_t last_master_deletion_us; + + void (*ins_callback)(const DICTIONARY_ITEM *item, void *value, void *data); + void *ins_callback_data; + + bool (*conflict_callback)(const DICTIONARY_ITEM *item, void *old_value, void *new_value, void *data); + void *conflict_callback_data; + + void (*react_callback)(const DICTIONARY_ITEM *item, void *value, void *data); + void *react_callback_data; + + void (*del_callback)(const DICTIONARY_ITEM *item, void *value, void *data); + void *del_callback_data; +}; + +struct dictionary_stats dictionary_stats_category_other = { + .name = "other", +}; struct dictionary { #ifdef NETDATA_INTERNAL_CHECKS @@ -52,332 +135,574 @@ struct dictionary { size_t creation_line; #endif - DICTIONARY_FLAGS flags; // the flags of the dictionary + usec_t last_gc_run_us; + DICT_OPTIONS options; // the configuration flags of the dictionary (they never change - no atomics) + DICT_FLAGS flags; // run time flags for the dictionary (they change all the time - atomics needed) - NAME_VALUE *first_item; // the double linked list base pointers - NAME_VALUE *last_item; + struct { // support for multiple indexing engines + Pvoid_t JudyHSArray; // the hash table + netdata_rwlock_t rwlock; // protect the index + } index; - Pvoid_t JudyHSArray; // the hash table + struct { + DICTIONARY_ITEM *list; // the double linked list of all items in the dictionary + netdata_rwlock_t rwlock; // protect the linked-list + pid_t writer_pid; // the gettid() of the writer + size_t writer_depth; // nesting of write locks + } items; - netdata_rwlock_t rwlock; // the r/w lock when DICTIONARY_FLAG_SINGLE_THREADED is not set + struct dictionary_hooks *hooks; // pointer to external function callbacks to be called at certain points + struct dictionary_stats *stats; // statistics data, when DICT_OPTION_STATS is set - void (*ins_callback)(const char *name, void *value, void *data); - void *ins_callback_data; + DICTIONARY *master; // the master dictionary + DICTIONARY *next; // linked list for delayed destruction (garbage collection of whole dictionaries) - void (*react_callback)(const char *name, void *value, void *data); - void *react_callback_data; + size_t version; // the current version of the dictionary + // it is incremented when: + // - item added + // - item removed + // - item value reset + // - conflict callback returns true + // - function dictionary_version_increment() is called - void (*del_callback)(const char *name, void *value, void *data); - void *del_callback_data; - - void (*conflict_callback)(const char *name, void *old_value, void *new_value, void *data); - void *conflict_callback_data; - - size_t version; // the current version of the dictionary - size_t inserts; // how many index insertions have been performed - size_t deletes; // how many index deletions have been performed - size_t searches; // how many index searches have been performed - size_t resets; // how many times items have reset their values - size_t walkthroughs; // how many walkthroughs have been done - long int memory; // how much memory the dictionary has currently allocated - long int entries; // how many items are currently in the index (the linked list may have more) - long int referenced_items; // how many items of the dictionary are currently being used by 3rd parties - long int pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet - int readers; // how many readers are currently using the dictionary - int writers; // how many writers are currently using the dictionary - - size_t scratchpad_size; // the size of the scratchpad in bytes - uint8_t scratchpad[]; // variable size scratchpad requested by the caller + long int entries; // how many items are currently in the index (the linked list may have more) + long int referenced_items; // how many items of the dictionary are currently being used by 3rd parties + long int pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet }; -static inline void linkedlist_namevalue_unlink_unsafe(DICTIONARY *dict, NAME_VALUE *nv); -static size_t namevalue_destroy_unsafe(DICTIONARY *dict, NAME_VALUE *nv); -static inline const char *namevalue_get_name(NAME_VALUE *nv); +// forward definitions of functions used in reverse order in the code +static void garbage_collect_pending_deletes(DICTIONARY *dict); +static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item); +static size_t item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item); +static inline const char *item_get_name(const DICTIONARY_ITEM *item); +static bool item_is_not_referenced_and_can_be_removed(DICTIONARY *dict, DICTIONARY_ITEM *item); +static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *item); +static void item_release(DICTIONARY *dict, DICTIONARY_ITEM *item); + +#define ITEM_OK 0 +#define ITEM_MARKED_FOR_DELETION (-1) // the item is marked for deletion +#define ITEM_IS_CURRENTLY_BEING_DELETED (-2) // the item is currently being deleted +#define item_check_and_acquire(dict, item) (item_check_and_acquire_advanced(dict, item, false) == ITEM_OK) +static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *item, bool having_index_lock); + +// ---------------------------------------------------------------------------- +// memory statistics + +static inline void DICTIONARY_STATS_PLUS_MEMORY(DICTIONARY *dict, size_t key_size, size_t item_size, size_t value_size) { + if(key_size) + __atomic_fetch_add(&dict->stats->memory.indexed, (long)key_size, __ATOMIC_RELAXED); + + if(item_size) + __atomic_fetch_add(&dict->stats->memory.dict, (long)item_size, __ATOMIC_RELAXED); + + if(value_size) + __atomic_fetch_add(&dict->stats->memory.values, (long)value_size, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_MINUS_MEMORY(DICTIONARY *dict, size_t key_size, size_t item_size, size_t value_size) { + if(key_size) + __atomic_fetch_sub(&dict->stats->memory.indexed, (long)key_size, __ATOMIC_RELAXED); + + if(item_size) + __atomic_fetch_sub(&dict->stats->memory.dict, (long)item_size, __ATOMIC_RELAXED); + + if(value_size) + __atomic_fetch_sub(&dict->stats->memory.values, (long)value_size, __ATOMIC_RELAXED); +} // ---------------------------------------------------------------------------- // callbacks registration -void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const char *name, void *value, void *data), void *data) { - dict->ins_callback = ins_callback; - dict->ins_callback_data = data; +static inline void dictionary_hooks_allocate(DICTIONARY *dict) { + if(dict->hooks) return; + + dict->hooks = callocz(1, sizeof(struct dictionary_hooks)); + dict->hooks->links = 1; + + DICTIONARY_STATS_PLUS_MEMORY(dict, 0, sizeof(struct dictionary_hooks), 0); } -void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_callback)(const char *name, void *value, void *data), void *data) { - dict->del_callback = del_callback; - dict->del_callback_data = data; +static inline size_t dictionary_hooks_free(DICTIONARY *dict) { + if(!dict->hooks) return 0; + + REFCOUNT links = __atomic_sub_fetch(&dict->hooks->links, 1, __ATOMIC_SEQ_CST); + if(links == 0) { + freez(dict->hooks); + dict->hooks = NULL; + + DICTIONARY_STATS_MINUS_MEMORY(dict, 0, sizeof(struct dictionary_hooks), 0); + return sizeof(struct dictionary_hooks); + } + + return 0; } -void dictionary_register_conflict_callback(DICTIONARY *dict, void (*conflict_callback)(const char *name, void *old_value, void *new_value, void *data), void *data) { - dict->conflict_callback = conflict_callback; - dict->conflict_callback_data = data; +void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data) { + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ ); + + dictionary_hooks_allocate(dict); + dict->hooks->ins_callback = ins_callback; + dict->hooks->ins_callback_data = data; } -void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)(const char *name, void *value, void *data), void *data) { - dict->react_callback = react_callback; - dict->react_callback_data = data; +void dictionary_register_conflict_callback(DICTIONARY *dict, bool (*conflict_callback)(const DICTIONARY_ITEM *item, void *old_value, void *new_value, void *data), void *data) { + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ ); + + dictionary_hooks_allocate(dict); + dict->hooks->conflict_callback = conflict_callback; + dict->hooks->conflict_callback_data = data; +} + +void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data) { + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ ); + + dictionary_hooks_allocate(dict); + dict->hooks->react_callback = react_callback; + dict->hooks->react_callback_data = data; +} + +void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data) { + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ ); + + dictionary_hooks_allocate(dict); + dict->hooks->del_callback = del_callback; + dict->hooks->del_callback_data = data; } // ---------------------------------------------------------------------------- -// dictionary statistics maintenance +// dictionary statistics API -long int dictionary_stats_allocated_memory(DICTIONARY *dict) { +size_t dictionary_version(DICTIONARY *dict) { if(unlikely(!dict)) return 0; - return dict->memory; + + // this is required for views to return the right number + garbage_collect_pending_deletes(dict); + + return __atomic_load_n(&dict->version, __ATOMIC_SEQ_CST); } -long int dictionary_stats_entries(DICTIONARY *dict) { +size_t dictionary_entries(DICTIONARY *dict) { if(unlikely(!dict)) return 0; - return dict->entries; + + // this is required for views to return the right number + garbage_collect_pending_deletes(dict); + + long int entries = __atomic_load_n(&dict->entries, __ATOMIC_SEQ_CST); + if(entries < 0) + fatal("DICTIONARY: entries is negative: %ld", entries); + + return entries; } -size_t dictionary_stats_version(DICTIONARY *dict) { +size_t dictionary_referenced_items(DICTIONARY *dict) { if(unlikely(!dict)) return 0; - return dict->version; + + long int referenced_items = __atomic_load_n(&dict->referenced_items, __ATOMIC_SEQ_CST); + if(referenced_items < 0) + fatal("DICTIONARY: referenced items is negative: %ld", referenced_items); + + return referenced_items; } -size_t dictionary_stats_searches(DICTIONARY *dict) { + +long int dictionary_stats_for_registry(DICTIONARY *dict) { if(unlikely(!dict)) return 0; - return dict->searches; + return (dict->stats->memory.indexed + dict->stats->memory.dict); } -size_t dictionary_stats_inserts(DICTIONARY *dict) { - if(unlikely(!dict)) return 0; - return dict->inserts; -} -size_t dictionary_stats_deletes(DICTIONARY *dict) { - if(unlikely(!dict)) return 0; - return dict->deletes; -} -size_t dictionary_stats_resets(DICTIONARY *dict) { - if(unlikely(!dict)) return 0; - return dict->resets; -} -size_t dictionary_stats_walkthroughs(DICTIONARY *dict) { - if(unlikely(!dict)) return 0; - return dict->walkthroughs; -} -size_t dictionary_stats_referenced_items(DICTIONARY *dict) { - if(unlikely(!dict)) return 0; - return __atomic_load_n(&dict->referenced_items, __ATOMIC_SEQ_CST); +void dictionary_version_increment(DICTIONARY *dict) { + __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST); } +// ---------------------------------------------------------------------------- +// internal statistics API + static inline void DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) { - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { - dict->searches++; - } - else { - __atomic_fetch_add(&dict->searches, 1, __ATOMIC_RELAXED); - } + __atomic_fetch_add(&dict->stats->ops.searches, 1, __ATOMIC_RELAXED); } -static inline void DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict, size_t size) { - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { +static inline void DICTIONARY_ENTRIES_PLUS1(DICTIONARY *dict) { + // statistics + __atomic_fetch_add(&dict->stats->items.entries, 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&dict->stats->ops.inserts, 1, __ATOMIC_RELAXED); + + if(unlikely(is_dictionary_single_threaded(dict))) { dict->version++; - dict->inserts++; dict->entries++; - dict->memory += (long)size; + dict->referenced_items++; + } else { __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST); - __atomic_fetch_add(&dict->inserts, 1, __ATOMIC_RELAXED); - __atomic_fetch_add(&dict->entries, 1, __ATOMIC_RELAXED); - __atomic_fetch_add(&dict->memory, (long)size, __ATOMIC_RELAXED); + __atomic_fetch_add(&dict->entries, 1, __ATOMIC_SEQ_CST); + __atomic_fetch_add(&dict->referenced_items, 1, __ATOMIC_SEQ_CST); } } -static inline void DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict) { - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { +static inline void DICTIONARY_ENTRIES_MINUS1(DICTIONARY *dict) { + // statistics + __atomic_fetch_add(&dict->stats->ops.deletes, 1, __ATOMIC_RELAXED); + __atomic_fetch_sub(&dict->stats->items.entries, 1, __ATOMIC_RELAXED); + + if(unlikely(is_dictionary_single_threaded(dict))) { dict->version++; - dict->deletes++; dict->entries--; } else { __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST); - __atomic_fetch_add(&dict->deletes, 1, __ATOMIC_RELAXED); - __atomic_fetch_sub(&dict->entries, 1, __ATOMIC_RELAXED); + __atomic_fetch_sub(&dict->entries, 1, __ATOMIC_SEQ_CST); } } -static inline void DICTIONARY_STATS_ENTRIES_MINUS_MEMORY(DICTIONARY *dict, size_t size) { - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { - dict->memory -= (long)size; - } - else { - __atomic_fetch_sub(&dict->memory, (long)size, __ATOMIC_RELAXED); - } -} -static inline void DICTIONARY_STATS_VALUE_RESETS_PLUS1(DICTIONARY *dict, size_t oldsize, size_t newsize) { - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { +static inline void DICTIONARY_VALUE_RESETS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->ops.resets, 1, __ATOMIC_RELAXED); + + if(unlikely(is_dictionary_single_threaded(dict))) dict->version++; - dict->resets++; - dict->memory += (long)newsize; - dict->memory -= (long)oldsize; - } - else { + else __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST); - __atomic_fetch_add(&dict->resets, 1, __ATOMIC_RELAXED); - __atomic_fetch_add(&dict->memory, (long)newsize, __ATOMIC_RELAXED); - __atomic_fetch_sub(&dict->memory, (long)oldsize, __ATOMIC_RELAXED); - } } - +static inline void DICTIONARY_STATS_TRAVERSALS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->ops.traversals, 1, __ATOMIC_RELAXED); +} static inline void DICTIONARY_STATS_WALKTHROUGHS_PLUS1(DICTIONARY *dict) { - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { - dict->walkthroughs++; - } - else { - __atomic_fetch_add(&dict->walkthroughs, 1, __ATOMIC_RELAXED); - } + __atomic_fetch_add(&dict->stats->ops.walkthroughs, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_CHECK_SPINS_PLUS(DICTIONARY *dict, size_t count) { + __atomic_fetch_add(&dict->stats->spin_locks.use, count, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_INSERT_SPINS_PLUS(DICTIONARY *dict, size_t count) { + __atomic_fetch_add(&dict->stats->spin_locks.insert, count, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_SEARCH_IGNORES_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->spin_locks.search, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->callbacks.inserts, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->callbacks.conflicts, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->callbacks.reacts, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->callbacks.deletes, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_GARBAGE_COLLECTIONS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->ops.garbage_collections, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_DICT_CREATIONS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->dictionaries.active, 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&dict->stats->ops.creations, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_DICT_DESTRUCTIONS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_sub(&dict->stats->dictionaries.active, 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&dict->stats->ops.destructions, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->dictionaries.deleted, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_DICT_DESTROY_QUEUED_MINUS1(DICTIONARY *dict) { + __atomic_fetch_sub(&dict->stats->dictionaries.deleted, 1, __ATOMIC_RELAXED); +} +static inline void DICTIONARY_STATS_DICT_FLUSHES_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->ops.flushes, 1, __ATOMIC_RELAXED); } -static inline size_t DICTIONARY_STATS_REFERENCED_ITEMS_PLUS1(DICTIONARY *dict) { - return __atomic_add_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST); +static inline long int DICTIONARY_REFERENCED_ITEMS_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED); + + if(unlikely(is_dictionary_single_threaded(dict))) + return ++dict->referenced_items; + else + return __atomic_add_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST); } -static inline size_t DICTIONARY_STATS_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) { - return __atomic_sub_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST); +static inline long int DICTIONARY_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) { + __atomic_fetch_sub(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED); + + if(unlikely(is_dictionary_single_threaded(dict))) + return --dict->referenced_items; + else + return __atomic_sub_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST); } -static inline size_t DICTIONARY_STATS_PENDING_DELETES_PLUS1(DICTIONARY *dict) { - return __atomic_add_fetch(&dict->pending_deletion_items, 1, __ATOMIC_SEQ_CST); +static inline long int DICTIONARY_PENDING_DELETES_PLUS1(DICTIONARY *dict) { + __atomic_fetch_add(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELAXED); + + if(unlikely(is_dictionary_single_threaded(dict))) + return ++dict->pending_deletion_items; + else + return __atomic_add_fetch(&dict->pending_deletion_items, 1, __ATOMIC_SEQ_CST); } -static inline size_t DICTIONARY_STATS_PENDING_DELETES_MINUS1(DICTIONARY *dict) { - return __atomic_sub_fetch(&dict->pending_deletion_items, 1, __ATOMIC_SEQ_CST); +static inline long int DICTIONARY_PENDING_DELETES_MINUS1(DICTIONARY *dict) { + __atomic_fetch_sub(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELAXED); + + if(unlikely(is_dictionary_single_threaded(dict))) + return --dict->pending_deletion_items; + else + return __atomic_sub_fetch(&dict->pending_deletion_items, 1, __ATOMIC_SEQ_CST); } -static inline size_t DICTIONARY_STATS_PENDING_DELETES_GET(DICTIONARY *dict) { - return __atomic_load_n(&dict->pending_deletion_items, __ATOMIC_SEQ_CST); +static inline long int DICTIONARY_PENDING_DELETES_GET(DICTIONARY *dict) { + if(unlikely(is_dictionary_single_threaded(dict))) + return dict->pending_deletion_items; + else + return __atomic_load_n(&dict->pending_deletion_items, __ATOMIC_SEQ_CST); } -static inline int DICTIONARY_NAME_VALUE_REFCOUNT_GET(NAME_VALUE *nv) { - return __atomic_load_n(&nv->refcount, __ATOMIC_SEQ_CST); +static inline REFCOUNT DICTIONARY_ITEM_REFCOUNT_GET(DICTIONARY *dict, DICTIONARY_ITEM *item) { + if(unlikely(dict && is_dictionary_single_threaded(dict))) // this is an exception, dict can be null + return item->refcount; + else + return (REFCOUNT)__atomic_load_n(&item->refcount, __ATOMIC_SEQ_CST); } // ---------------------------------------------------------------------------- -// garbage collector -// it is called every time someone gets a write lock to the dictionary +// callbacks execution -static void garbage_collect_pending_deletes_unsafe(DICTIONARY *dict) { - if(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS)) return; +static void dictionary_execute_insert_callback(DICTIONARY *dict, DICTIONARY_ITEM *item, void *constructor_data) { + if(likely(!dict->hooks || !dict->hooks->ins_callback)) + return; - if(likely(!DICTIONARY_STATS_PENDING_DELETES_GET(dict))) return; + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ ); - NAME_VALUE *nv = dict->first_item; - while(nv) { - if((nv->flags & NAME_VALUE_FLAG_DELETED) && DICTIONARY_NAME_VALUE_REFCOUNT_GET(nv) == 0) { - NAME_VALUE *nv_next = nv->next; + internal_error(false, + "DICTIONARY: Running insert callback on item '%s' of dictionary created from %s() %zu@%s.", + item_get_name(item), + dict->creation_function, + dict->creation_line, + dict->creation_file); - linkedlist_namevalue_unlink_unsafe(dict, nv); - namevalue_destroy_unsafe(dict, nv); + DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(dict); + dict->hooks->ins_callback(item, item->shared->value, constructor_data?constructor_data:dict->hooks->ins_callback_data); +} - size_t pending = DICTIONARY_STATS_PENDING_DELETES_MINUS1(dict); - if(!pending) break; +static bool dictionary_execute_conflict_callback(DICTIONARY *dict, DICTIONARY_ITEM *item, void *new_value, void *constructor_data) { + if(likely(!dict->hooks || !dict->hooks->conflict_callback)) + return false; - nv = nv_next; - } - else - nv = nv->next; - } + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ ); + + internal_error(false, + "DICTIONARY: Running conflict callback on item '%s' of dictionary created from %s() %zu@%s.", + item_get_name(item), + dict->creation_function, + dict->creation_line, + dict->creation_file); + + DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(dict); + return dict->hooks->conflict_callback( + item, item->shared->value, new_value, + constructor_data ? constructor_data : dict->hooks->conflict_callback_data); +} + +static void dictionary_execute_react_callback(DICTIONARY *dict, DICTIONARY_ITEM *item, void *constructor_data) { + if(likely(!dict->hooks || !dict->hooks->react_callback)) + return; + + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ ); + + internal_error(false, + "DICTIONARY: Running react callback on item '%s' of dictionary created from %s() %zu@%s.", + item_get_name(item), + dict->creation_function, + dict->creation_line, + dict->creation_file); + + DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(dict); + dict->hooks->react_callback(item, item->shared->value, + constructor_data?constructor_data:dict->hooks->react_callback_data); +} + +static void dictionary_execute_delete_callback(DICTIONARY *dict, DICTIONARY_ITEM *item) { + if(likely(!dict->hooks || !dict->hooks->del_callback)) + return; + + // We may execute the delete callback on items deleted from a view, + // because we may have references to it, after the master is gone + // so, the shared structure will remain until the last reference is released. + + internal_error(false, + "DICTIONARY: Running delete callback on item '%s' of dictionary created from %s() %zu@%s.", + item_get_name(item), + dict->creation_function, + dict->creation_line, + dict->creation_file); + + DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(dict); + dict->hooks->del_callback(item, item->shared->value, dict->hooks->del_callback_data); } // ---------------------------------------------------------------------------- // dictionary locks -static inline size_t dictionary_lock_init(DICTIONARY *dict) { - if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) { - netdata_rwlock_init(&dict->rwlock); - - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) - dict->flags &= ~DICTIONARY_FLAG_EXCLUSIVE_ACCESS; - - return 0; - } - - // we are single threaded - dict->flags |= DICTIONARY_FLAG_EXCLUSIVE_ACCESS; - return 0; -} - -static inline size_t dictionary_lock_free(DICTIONARY *dict) { - if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) { - netdata_rwlock_destroy(&dict->rwlock); +static inline size_t dictionary_locks_init(DICTIONARY *dict) { + if(likely(!is_dictionary_single_threaded(dict))) { + netdata_rwlock_init(&dict->index.rwlock); + netdata_rwlock_init(&dict->items.rwlock); return 0; } return 0; } -static void dictionary_lock(DICTIONARY *dict, char rw) { - if(rw == DICTIONARY_LOCK_NONE || rw == 'U') return; - - if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') { - // read lock - __atomic_add_fetch(&dict->readers, 1, __ATOMIC_RELAXED); - } - else { - // write lock - __atomic_add_fetch(&dict->writers, 1, __ATOMIC_RELAXED); +static inline size_t dictionary_locks_destroy(DICTIONARY *dict) { + if(likely(!is_dictionary_single_threaded(dict))) { + netdata_rwlock_destroy(&dict->index.rwlock); + netdata_rwlock_destroy(&dict->items.rwlock); + return 0; } + return 0; +} - if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) +static inline void ll_recursive_lock_set_thread_as_writer(DICTIONARY *dict) { + pid_t expected = 0, desired = gettid(); + if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) + fatal("DICTIONARY: Cannot set thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_SEQ_CST)); +} + +static inline void ll_recursive_unlock_unset_thread_writer(DICTIONARY *dict) { + pid_t expected = gettid(), desired = 0; + if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) + fatal("DICTIONARY: Cannot unset thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_SEQ_CST)); +} + +static inline bool ll_recursive_lock_is_thread_the_writer(DICTIONARY *dict) { + pid_t tid = gettid(); + return tid > 0 && tid == __atomic_load_n(&dict->items.writer_pid, __ATOMIC_SEQ_CST); +} + +static void ll_recursive_lock(DICTIONARY *dict, char rw) { + if(unlikely(is_dictionary_single_threaded(dict))) return; + if(ll_recursive_lock_is_thread_the_writer(dict)) { + dict->items.writer_depth++; + return; + } + if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') { // read lock - netdata_rwlock_rdlock(&dict->rwlock); - - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { - internal_error(true, "DICTIONARY: left-over exclusive access to dictionary created by %s (%zu@%s) found", dict->creation_function, dict->creation_line, dict->creation_file); - dict->flags &= ~DICTIONARY_FLAG_EXCLUSIVE_ACCESS; - } + netdata_rwlock_rdlock(&dict->items.rwlock); } else { // write lock - netdata_rwlock_wrlock(&dict->rwlock); - - dict->flags |= DICTIONARY_FLAG_EXCLUSIVE_ACCESS; + netdata_rwlock_wrlock(&dict->items.rwlock); + ll_recursive_lock_set_thread_as_writer(dict); } } -static void dictionary_unlock(DICTIONARY *dict, char rw) { - if(rw == DICTIONARY_LOCK_NONE || rw == 'U') return; +static void ll_recursive_unlock(DICTIONARY *dict, char rw) { + if(unlikely(is_dictionary_single_threaded(dict))) + return; + + if(ll_recursive_lock_is_thread_the_writer(dict) && dict->items.writer_depth > 0) { + dict->items.writer_depth--; + return; + } if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') { // read unlock - __atomic_sub_fetch(&dict->readers, 1, __ATOMIC_RELAXED); + + netdata_rwlock_unlock(&dict->items.rwlock); } else { // write unlock - garbage_collect_pending_deletes_unsafe(dict); - __atomic_sub_fetch(&dict->writers, 1, __ATOMIC_RELAXED); - } - if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) + ll_recursive_unlock_unset_thread_writer(dict); + + netdata_rwlock_unlock(&dict->items.rwlock); + } +} + + +static inline void dictionary_index_lock_rdlock(DICTIONARY *dict) { + if(unlikely(is_dictionary_single_threaded(dict))) return; - if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) - dict->flags &= ~DICTIONARY_FLAG_EXCLUSIVE_ACCESS; + netdata_rwlock_rdlock(&dict->index.rwlock); +} +static inline void dictionary_index_lock_wrlock(DICTIONARY *dict) { + if(unlikely(is_dictionary_single_threaded(dict))) + return; - netdata_rwlock_unlock(&dict->rwlock); + netdata_rwlock_wrlock(&dict->index.rwlock); +} +static inline void dictionary_index_lock_unlock(DICTIONARY *dict) { + if(unlikely(is_dictionary_single_threaded(dict))) + return; + + netdata_rwlock_unlock(&dict->index.rwlock); } // ---------------------------------------------------------------------------- -// deferred deletions +// items garbage collector -void dictionary_defer_all_deletions_unsafe(DICTIONARY *dict, char rw) { - if(rw == 'r' || rw == 'R') { - // read locked - no need to defer deletions - ; - } - else { - // write locked - defer deletions - dict->flags |= DICTIONARY_FLAG_DEFER_ALL_DELETIONS; - } -} +static void garbage_collect_pending_deletes(DICTIONARY *dict) { + usec_t last_master_deletion_us = dict->hooks?__atomic_load_n(&dict->hooks->last_master_deletion_us, __ATOMIC_SEQ_CST):0; + usec_t last_gc_run_us = __atomic_load_n(&dict->last_gc_run_us, __ATOMIC_SEQ_CST); -void dictionary_restore_all_deletions_unsafe(DICTIONARY *dict, char rw) { - if(rw == 'r' || rw == 'R') { - // read locked - no need to defer deletions - internal_error(dict->flags & DICTIONARY_FLAG_DEFER_ALL_DELETIONS, "DICTIONARY: deletions are deferred on a read lock"); - } - else { - // write locked - defer deletions - if(dict->flags & DICTIONARY_FLAG_DEFER_ALL_DELETIONS) - dict->flags &= ~DICTIONARY_FLAG_DEFER_ALL_DELETIONS; + bool is_view = is_view_dictionary(dict); + + if(likely(!( + DICTIONARY_PENDING_DELETES_GET(dict) > 0 || + (is_view && last_master_deletion_us > last_gc_run_us) + ))) + return; + + ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); + + __atomic_store_n(&dict->last_gc_run_us, now_realtime_usec(), __ATOMIC_SEQ_CST); + + if(is_view) + dictionary_index_lock_wrlock(dict); + + DICTIONARY_STATS_GARBAGE_COLLECTIONS_PLUS1(dict); + + size_t deleted = 0, pending = 0, examined = 0; + DICTIONARY_ITEM *item = dict->items.list, *item_next; + while(item) { + examined++; + + // this will cleanup + item_next = item->next; + int rc = item_check_and_acquire_advanced(dict, item, is_view); + + if(rc == ITEM_MARKED_FOR_DELETION) { + // we don't have got a reference + + if(item_is_not_referenced_and_can_be_removed(dict, item)) { + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next); + item_free_with_hooks(dict, item); + deleted++; + + pending = DICTIONARY_PENDING_DELETES_MINUS1(dict); + if (!pending) + break; + } + } + else if(rc == ITEM_IS_CURRENTLY_BEING_DELETED) + ; // do not touch this item (we haven't got a reference) + + else if(rc == ITEM_OK) + item_release(dict, item); + + item = item_next; } + + if(is_view) + dictionary_index_lock_unlock(dict); + + ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE); + + (void)deleted; + (void)examined; + + internal_error(false, "DICTIONARY: garbage collected dictionary created by %s (%zu@%s), examined %zu items, deleted %zu items, still pending %zu items", + dict->creation_function, dict->creation_line, dict->creation_file, examined, deleted, pending); + } // ---------------------------------------------------------------------------- @@ -399,82 +724,208 @@ static inline size_t reference_counter_free(DICTIONARY *dict) { return 0; } -static int reference_counter_increase(NAME_VALUE *nv) { - int refcount = __atomic_add_fetch(&nv->refcount, 1, __ATOMIC_SEQ_CST); - if(refcount == 1) - fatal("DICTIONARY: request to dup item '%s' but its reference counter was zero", namevalue_get_name(nv)); - return refcount; -} +static void item_acquire(DICTIONARY *dict, DICTIONARY_ITEM *item) { + REFCOUNT refcount; -static int reference_counter_acquire(DICTIONARY *dict, NAME_VALUE *nv) { - int refcount; - if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) - refcount = ++nv->refcount; - else - refcount = __atomic_add_fetch(&nv->refcount, 1, __ATOMIC_SEQ_CST); + if(unlikely(is_dictionary_single_threaded(dict))) { + refcount = ++item->refcount; + } + else { + // increment the refcount + refcount = __atomic_add_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST); + } + + if(refcount <= 0) { + internal_error( + true, + "DICTIONARY: attempted to acquire item which is deleted (refcount = %d): " + "'%s' on dictionary created by %s() (%zu@%s)", + refcount - 1, + item_get_name(item), + dict->creation_function, + dict->creation_line, + dict->creation_file); + + fatal( + "DICTIONARY: request to acquire item '%s', which is deleted (refcount = %d)!", + item_get_name(item), + refcount - 1); + } if(refcount == 1) { // referenced items counts number of unique items referenced // so, we increase it only when refcount == 1 - DICTIONARY_STATS_REFERENCED_ITEMS_PLUS1(dict); + DICTIONARY_REFERENCED_ITEMS_PLUS1(dict); // if this is a deleted item, but the counter increased to 1 // we need to remove it from the pending items to delete - if (nv->flags & NAME_VALUE_FLAG_DELETED) - DICTIONARY_STATS_PENDING_DELETES_MINUS1(dict); + if(item_flag_check(item, ITEM_FLAG_DELETED)) + DICTIONARY_PENDING_DELETES_MINUS1(dict); } - - return refcount; } -static uint32_t reference_counter_release(DICTIONARY *dict, NAME_VALUE *nv, bool can_get_write_lock) { +static void item_release(DICTIONARY *dict, DICTIONARY_ITEM *item) { // this function may be called without any lock on the dictionary - // or even when someone else has a write lock on the dictionary - // so, we cannot check for EXCLUSIVE ACCESS + // or even when someone else has write lock on the dictionary - uint32_t refcount; - if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) - refcount = nv->refcount--; - else - refcount = __atomic_fetch_sub(&nv->refcount, 1, __ATOMIC_SEQ_CST); + bool is_deleted; + REFCOUNT refcount; - if(refcount == 0) { - internal_error(true, "DICTIONARY: attempted to release item without references: '%s' on dictionary created by %s() (%zu@%s)", namevalue_get_name(nv), dict->creation_function, dict->creation_line, dict->creation_file); - fatal("DICTIONARY: attempted to release item without references: '%s'", namevalue_get_name(nv)); + if(unlikely(is_dictionary_single_threaded(dict))) { + is_deleted = item->flags & ITEM_FLAG_DELETED; + refcount = --item->refcount; + } + else { + // get the flags before decrementing any reference counters + // (the other way around may lead to use-after-free) + is_deleted = item_flag_check(item, ITEM_FLAG_DELETED); + + // decrement the refcount + refcount = __atomic_sub_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST); } - if(refcount == 1) { - if((nv->flags & NAME_VALUE_FLAG_DELETED)) - DICTIONARY_STATS_PENDING_DELETES_PLUS1(dict); + if(refcount < 0) { + internal_error( + true, + "DICTIONARY: attempted to release item without references (refcount = %d): " + "'%s' on dictionary created by %s() (%zu@%s)", + refcount + 1, + item_get_name(item), + dict->creation_function, + dict->creation_line, + dict->creation_file); + + fatal( + "DICTIONARY: attempted to release item '%s' without references (refcount = %d)", + item_get_name(item), + refcount + 1); + } + + if(refcount == 0) { + + if(is_deleted) + DICTIONARY_PENDING_DELETES_PLUS1(dict); // referenced items counts number of unique items referenced // so, we decrease it only when refcount == 0 - DICTIONARY_STATS_REFERENCED_ITEMS_MINUS1(dict); + DICTIONARY_REFERENCED_ITEMS_MINUS1(dict); } - - if(can_get_write_lock && DICTIONARY_STATS_PENDING_DELETES_GET(dict)) { - // we can garbage collect now - - dictionary_lock(dict, DICTIONARY_LOCK_WRITE); - garbage_collect_pending_deletes_unsafe(dict); - dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); - } - - return refcount; } -// ---------------------------------------------------------------------------- -// hash table +static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *item, bool having_index_lock) { + size_t spins = 0; + REFCOUNT refcount, desired; -static void hashtable_init_unsafe(DICTIONARY *dict) { - dict->JudyHSArray = NULL; + do { + spins++; + + refcount = DICTIONARY_ITEM_REFCOUNT_GET(dict, item); + + if(refcount < 0) { + // we can't use this item + return ITEM_IS_CURRENTLY_BEING_DELETED; + } + + if(item_flag_check(item, ITEM_FLAG_DELETED)) { + // we can't use this item + return ITEM_MARKED_FOR_DELETION; + } + + desired = refcount + 1; + + } while(!__atomic_compare_exchange_n(&item->refcount, &refcount, desired, + false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)); + + // we acquired the item + + if(is_view_dictionary(dict) && item_shared_flag_check(item, ITEM_FLAG_DELETED) && !item_flag_check(item, ITEM_FLAG_DELETED)) { + // but, we can't use this item + + if(having_index_lock) { + // delete it from the hashtable + hashtable_delete_unsafe(dict, item_get_name(item), item->key_len, item); + + // mark it in our dictionary as deleted too + // this is safe to be done here, because we have got + // a reference counter on item + item_flag_set(item, ITEM_FLAG_DELETED); + + DICTIONARY_ENTRIES_MINUS1(dict); + + // decrement the refcount we incremented above + if (__atomic_sub_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST) == 0) { + // this is a deleted item, and we are the last one + DICTIONARY_PENDING_DELETES_PLUS1(dict); + } + + // do not touch the item below this point + } + else { + // this is traversal / walkthrough + // decrement the refcount we incremented above + __atomic_sub_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST); + } + + return ITEM_MARKED_FOR_DELETION; + } + + if(desired == 1) + DICTIONARY_REFERENCED_ITEMS_PLUS1(dict); + + if(unlikely(spins > 2 && dict->stats)) + DICTIONARY_STATS_CHECK_SPINS_PLUS(dict, spins - 2); + + return ITEM_OK; // we can use this item +} + +// if a dictionary item can be deleted, return true, otherwise return false +// we use the private reference counter +static inline bool item_is_not_referenced_and_can_be_removed(DICTIONARY *dict, DICTIONARY_ITEM *item) { + // if we can set refcount to REFCOUNT_DELETING, we can delete this item + + REFCOUNT expected = DICTIONARY_ITEM_REFCOUNT_GET(dict, item); + if(expected == 0 && __atomic_compare_exchange_n(&item->refcount, &expected, REFCOUNT_DELETING, + false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + + // we are going to delete it + return true; + } + + // we can't delete this + return false; +} + +// if a dictionary item can be freed, return true, otherwise return false +// we use the shared reference counter +static inline bool item_shared_release_and_check_if_it_can_be_freed(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item) { + // if we can set refcount to REFCOUNT_DELETING, we can delete this item + + REFCOUNT links = __atomic_sub_fetch(&item->shared->links, 1, __ATOMIC_SEQ_CST); + if(links == 0 && __atomic_compare_exchange_n(&item->shared->links, &links, REFCOUNT_DELETING, + false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + + // we can delete it + return true; + } + + // we can't delete it + return false; +} + + +// ---------------------------------------------------------------------------- +// hash table operations + +static size_t hashtable_init_unsafe(DICTIONARY *dict) { + dict->index.JudyHSArray = NULL; + return 0; } static size_t hashtable_destroy_unsafe(DICTIONARY *dict) { - if(unlikely(!dict->JudyHSArray)) return 0; + if(unlikely(!dict->index.JudyHSArray)) return 0; JError_t J_Error; - Word_t ret = JudyHSFreeArray(&dict->JudyHSArray, &J_Error); + Word_t ret = JudyHSFreeArray(&dict->index.JudyHSArray, &J_Error); if(unlikely(ret == (Word_t) JERR)) { error("DICTIONARY: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d", JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); @@ -482,15 +933,13 @@ static size_t hashtable_destroy_unsafe(DICTIONARY *dict) { debug(D_DICTIONARY, "Dictionary: hash table freed %lu bytes", ret); - dict->JudyHSArray = NULL; + dict->index.JudyHSArray = NULL; return (size_t)ret; } static inline void **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { - internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: inserting item from the index without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); - JError_t J_Error; - Pvoid_t *Rc = JudyHSIns(&dict->JudyHSArray, (void *)name, name_len, &J_Error); + Pvoid_t *Rc = JudyHSIns(&dict->index.JudyHSArray, (void *)name, name_len, &J_Error); if (unlikely(Rc == PJERR)) { fatal("DICTIONARY: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d", name, JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); @@ -506,14 +955,12 @@ static inline void **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, return Rc; } -static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *nv) { - internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: deleting item from the index without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); - - (void)nv; - if(unlikely(!dict->JudyHSArray)) return 0; +static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *item) { + (void)item; + if(unlikely(!dict->index.JudyHSArray)) return 0; JError_t J_Error; - int ret = JudyHSDel(&dict->JudyHSArray, (void *)name, name_len, &J_Error); + int ret = JudyHSDel(&dict->index.JudyHSArray, (void *)name, name_len, &J_Error); if(unlikely(ret == JERR)) { error("DICTIONARY: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d", name, JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); @@ -533,16 +980,16 @@ static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, si } } -static inline NAME_VALUE *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { - if(unlikely(!dict->JudyHSArray)) return NULL; +static inline DICTIONARY_ITEM *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { + if(unlikely(!dict->index.JudyHSArray)) return NULL; DICTIONARY_STATS_SEARCHES_PLUS1(dict); Pvoid_t *Rc; - Rc = JudyHSGet(dict->JudyHSArray, (void *)name, name_len); + Rc = JudyHSGet(dict->index.JudyHSArray, (void *)name, name_len); if(likely(Rc)) { // found in the hash table - return (NAME_VALUE *)*Rc; + return (DICTIONARY_ITEM *)*Rc; } else { // not found in the hash table @@ -550,343 +997,352 @@ static inline NAME_VALUE *hashtable_get_unsafe(DICTIONARY *dict, const char *nam } } -static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, void *nv) { +static inline void hashtable_inserted_item_unsafe(DICTIONARY *dict, void *item) { (void)dict; - (void)nv; + (void)item; + + // this is called just after an item is successfully inserted to the hashtable + // we don't need this for judy, but we may need it if we integrate more hash tables + ; } // ---------------------------------------------------------------------------- // linked list management -static inline void linkedlist_namevalue_link_unsafe(DICTIONARY *dict, NAME_VALUE *nv) { - internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: adding item to the linked-list without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); +static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item) { + ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); - if (unlikely(!dict->first_item)) { - // we are the only ones here - nv->next = NULL; - nv->prev = NULL; - dict->first_item = dict->last_item = nv; - return; - } + if(dict->options & DICT_OPTION_ADD_IN_FRONT) + DOUBLE_LINKED_LIST_PREPEND_UNSAFE(dict->items.list, item, prev, next); + else + DOUBLE_LINKED_LIST_APPEND_UNSAFE(dict->items.list, item, prev, next); - if(dict->flags & DICTIONARY_FLAG_ADD_IN_FRONT) { - // add it at the beginning - nv->prev = NULL; - nv->next = dict->first_item; - - if (likely(nv->next)) nv->next->prev = nv; - dict->first_item = nv; - } - else { - // add it at the end - nv->next = NULL; - nv->prev = dict->last_item; - - if (likely(nv->prev)) nv->prev->next = nv; - dict->last_item = nv; - } + garbage_collect_pending_deletes(dict); + ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE); } -static inline void linkedlist_namevalue_unlink_unsafe(DICTIONARY *dict, NAME_VALUE *nv) { - internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: removing item from the linked-list without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); +static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item) { + ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); - if(nv->next) nv->next->prev = nv->prev; - if(nv->prev) nv->prev->next = nv->next; - if(dict->first_item == nv) dict->first_item = nv->next; - if(dict->last_item == nv) dict->last_item = nv->prev; + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next); + + garbage_collect_pending_deletes(dict); + ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE); } // ---------------------------------------------------------------------------- -// NAME_VALUE methods +// ITEM initialization and updates -static inline size_t namevalue_set_name(DICTIONARY *dict, NAME_VALUE *nv, const char *name, size_t name_len) { - if(likely(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)) { - nv->caller_name = (char *)name; - return 0; +static inline size_t item_set_name(DICTIONARY *dict, DICTIONARY_ITEM *item, const char *name, size_t name_len) { + if(likely(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE)) { + item->caller_name = (char *)name; + item->key_len = name_len; + } + else { + item->string_name = string_strdupz(name); + item->key_len = string_strlen(item->string_name) + 1; + item->options |= ITEM_OPTION_ALLOCATED_NAME; } - nv->string_name = string_strdupz(name); - nv->flags |= NAME_VALUE_FLAG_NAME_IS_ALLOCATED; - return name_len; + return item->key_len; } -static inline size_t namevalue_free_name(DICTIONARY *dict, NAME_VALUE *nv) { - if(unlikely(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))) - string_freez(nv->string_name); +static inline size_t item_free_name(DICTIONARY *dict, DICTIONARY_ITEM *item) { + if(likely(!(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE))) + string_freez(item->string_name); - return 0; + return item->key_len; } -static inline const char *namevalue_get_name(NAME_VALUE *nv) { - if(nv->flags & NAME_VALUE_FLAG_NAME_IS_ALLOCATED) - return string2str(nv->string_name); +static inline const char *item_get_name(const DICTIONARY_ITEM *item) { + if(item->options & ITEM_OPTION_ALLOCATED_NAME) + return string2str(item->string_name); else - return nv->caller_name; + return item->caller_name; } -static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len) { - debug(D_DICTIONARY, "Creating name value entry for name '%s'.", name); +static DICTIONARY_ITEM *item_allocate(DICTIONARY *dict __maybe_unused, size_t *allocated_bytes, DICTIONARY_ITEM *master_item) { + DICTIONARY_ITEM *item; - size_t size = sizeof(NAME_VALUE); - NAME_VALUE *nv = mallocz(size); - size_t allocated = size; + size_t size = sizeof(DICTIONARY_ITEM); + item = callocz(1, size); + item->refcount = 1; + *allocated_bytes += size; + + if(master_item) { + item->shared = master_item->shared; + + if(unlikely(__atomic_add_fetch(&item->shared->links, 1, __ATOMIC_SEQ_CST) <= 1)) + fatal("DICTIONARY: attempted to link to a shared item structure that had zero references"); + } + else { + size = sizeof(DICTIONARY_ITEM_SHARED); + item->shared = callocz(1, size); + item->shared->links = 1; + *allocated_bytes += size; + } #ifdef NETDATA_INTERNAL_CHECKS - nv->dict = dict; + item->dict = dict; +#endif + return item; +} + +static DICTIONARY_ITEM *item_create_with_hooks(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) { +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(name_len > KEY_LEN_MAX)) + fatal("DICTIONARY: tried to index a key of size %zu, but the maximum acceptable is %zu", name_len, (size_t)KEY_LEN_MAX); + + if(unlikely(value_len > VALUE_LEN_MAX)) + fatal("DICTIONARY: tried to add an item of size %zu, but the maximum acceptable is %zu", value_len, (size_t)VALUE_LEN_MAX); #endif - nv->refcount = 0; - nv->flags = NAME_VALUE_FLAG_NONE; - nv->value_len = value_len; + size_t item_size = 0, key_size = 0, value_size = 0; - allocated += namevalue_set_name(dict, nv, name, name_len); + DICTIONARY_ITEM *item = item_allocate(dict, &item_size, master_item); + key_size += item_set_name(dict, item, name, name_len); - if(likely(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) - nv->value = value; + if(unlikely(is_view_dictionary(dict))) { + // we are on a view dictionary + // do not touch the value + ; + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(!master_item)) + fatal("DICTIONARY: cannot add an item to a view without a master item."); +#endif + } else { - if(likely(value_len)) { - if (value) { - // a value has been supplied - // copy it - nv->value = mallocz(value_len); - memcpy(nv->value, value, value_len); + // we are on the master dictionary + + if(likely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE)) + item->shared->value = value; + else { + if(likely(value_len)) { + if(value) { + // a value has been supplied + // copy it + item->shared->value = mallocz(value_len); + memcpy(item->shared->value, value, value_len); + } + else { + // no value has been supplied + // allocate a clear memory block + item->shared->value = callocz(1, value_len); + } + } else { - // no value has been supplied - // allocate a clear memory block - nv->value = callocz(1, value_len); + // the caller wants an item without any value + item->shared->value = NULL; } } - else { - // the caller wants an item without any value - nv->value = NULL; - } + item->shared->value_len = value_len; + value_size += value_len; - allocated += value_len; + dictionary_execute_insert_callback(dict, item, constructor_data); } - DICTIONARY_STATS_ENTRIES_PLUS1(dict, allocated); + DICTIONARY_ENTRIES_PLUS1(dict); + DICTIONARY_STATS_PLUS_MEMORY(dict, key_size, item_size, value_size); - if(dict->ins_callback) - dict->ins_callback(namevalue_get_name(nv), nv->value, dict->ins_callback_data); - - return nv; + return item; } -static void namevalue_reset_unsafe(DICTIONARY *dict, NAME_VALUE *nv, void *value, size_t value_len) { - debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", namevalue_get_name(nv)); +static void item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item, void *value, size_t value_len, void *constructor_data) { + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: %s() should never be called on views.", __FUNCTION__ ); - DICTIONARY_STATS_VALUE_RESETS_PLUS1(dict, nv->value_len, value_len); + debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", item_get_name(item)); - if(dict->del_callback) - dict->del_callback(namevalue_get_name(nv), nv->value, dict->del_callback_data); + DICTIONARY_VALUE_RESETS_PLUS1(dict); - if(likely(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) { - debug(D_DICTIONARY, "Dictionary: linking value to '%s'", namevalue_get_name(nv)); - nv->value = value; - nv->value_len = value_len; + if(item->shared->value_len != value_len) { + DICTIONARY_STATS_PLUS_MEMORY(dict, 0, 0, value_len); + DICTIONARY_STATS_MINUS_MEMORY(dict, 0, 0, item->shared->value_len); + } + + dictionary_execute_delete_callback(dict, item); + + if(likely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE)) { + debug(D_DICTIONARY, "Dictionary: linking value to '%s'", item_get_name(item)); + item->shared->value = value; + item->shared->value_len = value_len; } else { - debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", namevalue_get_name(nv)); + debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", item_get_name(item)); - void *oldvalue = nv->value; - void *newvalue = NULL; + void *old_value = item->shared->value; + void *new_value = NULL; if(value_len) { - newvalue = mallocz(value_len); - if(value) memcpy(newvalue, value, value_len); - else memset(newvalue, 0, value_len); + new_value = mallocz(value_len); + if(value) memcpy(new_value, value, value_len); + else memset(new_value, 0, value_len); } - nv->value = newvalue; - nv->value_len = value_len; + item->shared->value = new_value; + item->shared->value_len = value_len; - debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", namevalue_get_name(nv)); - freez(oldvalue); + debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", item_get_name(item)); + freez(old_value); } - if(dict->ins_callback) - dict->ins_callback(namevalue_get_name(nv), nv->value, dict->ins_callback_data); + dictionary_execute_insert_callback(dict, item, constructor_data); } -static size_t namevalue_destroy_unsafe(DICTIONARY *dict, NAME_VALUE *nv) { - debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", namevalue_get_name(nv)); +static size_t item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) { + debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", item_get_name(item)); - if(dict->del_callback) - dict->del_callback(namevalue_get_name(nv), nv->value, dict->del_callback_data); + size_t item_size = 0, key_size = 0, value_size = 0; - size_t freed = 0; + key_size += item->key_len; + if(unlikely(!(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE))) + item_free_name(dict, item); - if(unlikely(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE))) { - debug(D_DICTIONARY, "Dictionary freeing value of '%s'", namevalue_get_name(nv)); - freez(nv->value); - freed += nv->value_len; + if(item_shared_release_and_check_if_it_can_be_freed(dict, item)) { + dictionary_execute_delete_callback(dict, item); + + if(unlikely(!(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))) { + debug(D_DICTIONARY, "Dictionary freeing value of '%s'", item_get_name(item)); + freez(item->shared->value); + item->shared->value = NULL; + } + value_size += item->shared->value_len; + + freez(item->shared); + item->shared = NULL; + item_size += sizeof(DICTIONARY_ITEM_SHARED); } - if(unlikely(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))) { - debug(D_DICTIONARY, "Dictionary freeing name '%s'", namevalue_get_name(nv)); - freed += namevalue_free_name(dict, nv); - } + freez(item); + item_size += sizeof(DICTIONARY_ITEM); - freez(nv); - freed += sizeof(NAME_VALUE); + DICTIONARY_STATS_MINUS_MEMORY(dict, key_size, item_size, value_size); - DICTIONARY_STATS_ENTRIES_MINUS_MEMORY(dict, freed); - - return freed; -} - -// if a dictionary item can be deleted, return true, otherwise return false -static bool name_value_can_be_deleted(DICTIONARY *dict, NAME_VALUE *nv) { - if(unlikely(dict->flags & DICTIONARY_FLAG_DEFER_ALL_DELETIONS)) - return false; - - if(unlikely(DICTIONARY_NAME_VALUE_REFCOUNT_GET(nv) > 0)) - return false; - - return true; + // we return the memory we actually freed + return item_size + (dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE)?0:value_size; } // ---------------------------------------------------------------------------- -// API - dictionary management -#ifdef NETDATA_INTERNAL_CHECKS -DICTIONARY *dictionary_create_advanced_with_trace(DICTIONARY_FLAGS flags, size_t scratchpad_size, const char *function, size_t line, const char *file) { -#else -DICTIONARY *dictionary_create_advanced(DICTIONARY_FLAGS flags, size_t scratchpad_size) { -#endif - debug(D_DICTIONARY, "Creating dictionary."); +// item operations - if(unlikely(flags & DICTIONARY_FLAGS_RESERVED)) - flags &= ~DICTIONARY_FLAGS_RESERVED; +static void item_shared_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) { + if(is_master_dictionary(dict)) { + item_shared_flag_set(item, ITEM_FLAG_DELETED); - DICTIONARY *dict = callocz(1, sizeof(DICTIONARY) + scratchpad_size); - size_t allocated = sizeof(DICTIONARY) + scratchpad_size; - - dict->scratchpad_size = scratchpad_size; - dict->flags = flags; - dict->first_item = dict->last_item = NULL; - - allocated += dictionary_lock_init(dict); - allocated += reference_counter_init(dict); - dict->memory = (long)allocated; - - hashtable_init_unsafe(dict); - -#ifdef NETDATA_INTERNAL_CHECKS - dict->creation_function = function; - dict->creation_file = file; - dict->creation_line = line; -#endif - - return (DICTIONARY *)dict; + if(dict->hooks) + __atomic_store_n(&dict->hooks->last_master_deletion_us, now_realtime_usec(), __ATOMIC_SEQ_CST); + } } -void *dictionary_scratchpad(DICTIONARY *dict) { - return &dict->scratchpad; +static inline void item_free_or_mark_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) { + if(item_is_not_referenced_and_can_be_removed(dict, item)) { + item_shared_set_deleted(dict, item); + item_linked_list_remove(dict, item); + item_free_with_hooks(dict, item); + } + else { + item_shared_set_deleted(dict, item); + item_flag_set(item, ITEM_FLAG_DELETED); + // after this point do not touch the item + } + + // the item is not available anymore + DICTIONARY_ENTRIES_MINUS1(dict); } -size_t dictionary_destroy(DICTIONARY *dict) { - if(!dict) return 0; +// this is used by traversal functions to remove the current item +// if it is deleted and it has zero references. This will eliminate +// the need for the garbage collector to kick-in later. +// Most deletions happen during traversal, so this is a nice hack +// to speed up everything! +static inline void item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(DICTIONARY *dict, DICTIONARY_ITEM *item, char rw) { + if(rw == DICTIONARY_LOCK_WRITE) { + bool should_be_deleted = item_flag_check(item, ITEM_FLAG_DELETED); - NAME_VALUE *nv; + item_release(dict, item); - debug(D_DICTIONARY, "Destroying dictionary."); + if(should_be_deleted && item_is_not_referenced_and_can_be_removed(dict, item)) { + // this has to be before removing from the linked list, + // otherwise the garbage collector will also kick in! + DICTIONARY_PENDING_DELETES_MINUS1(dict); - long referenced_items = 0; - size_t retries = 0; - do { - referenced_items = __atomic_load_n(&dict->referenced_items, __ATOMIC_SEQ_CST); - if (referenced_items) { - dictionary_lock(dict, DICTIONARY_LOCK_WRITE); - - // there are referenced items - // delete all items individually, so that only the referenced will remain - NAME_VALUE *nv_next; - for (nv = dict->first_item; nv; nv = nv_next) { - nv_next = nv->next; - size_t refcount = DICTIONARY_NAME_VALUE_REFCOUNT_GET(nv); - if (!refcount && !(nv->flags & NAME_VALUE_FLAG_DELETED)) - dictionary_del_unsafe(dict, namevalue_get_name(nv)); - } - - internal_error( - retries == 0, - "DICTIONARY: waiting (try %zu) for destruction of dictionary created from %s() %zu@%s, because it has %ld referenced items in it (%ld total).", - retries + 1, - dict->creation_function, - dict->creation_line, - dict->creation_file, - referenced_items, - dict->entries); - - dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); - sleep_usec(10000); + item_linked_list_remove(dict, item); + item_free_with_hooks(dict, item); } - } while(referenced_items > 0 && ++retries < 10); + } + else { + // we can't do anything under this mode + item_release(dict, item); + } +} - if(referenced_items) { - dictionary_lock(dict, DICTIONARY_LOCK_WRITE); - - dict->flags |= DICTIONARY_FLAG_DESTROYED; +static bool item_del(DICTIONARY *dict, const char *name, ssize_t name_len) { + if(unlikely(!name || !*name)) { internal_error( true, - "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s after %zu retries, because it has %ld referenced items in it (%ld total).", + "DICTIONARY: attempted to %s() without a name on a dictionary created from %s() %zu@%s.", + __FUNCTION__, dict->creation_function, dict->creation_line, - dict->creation_file, - retries, - referenced_items, - dict->entries); - - dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); - return 0; + dict->creation_file); + return false; } - dictionary_lock(dict, DICTIONARY_LOCK_WRITE); - - size_t freed = 0; - nv = dict->first_item; - while (nv) { - // cache nv->next - // because we are going to free nv - NAME_VALUE *nv_next = nv->next; - freed += namevalue_destroy_unsafe(dict, nv); - nv = nv_next; - // to speed up destruction, we don't - // unlink nv from the linked-list here + if(unlikely(is_dictionary_destroyed(dict))) { + internal_error(true, "DICTIONARY: attempted to dictionary_del() on a destroyed dictionary"); + return false; } - dict->first_item = NULL; - dict->last_item = NULL; + if(name_len == -1) + name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too - // destroy the dictionary - freed += hashtable_destroy_unsafe(dict); + debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name); - dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); - freed += dictionary_lock_free(dict); - freed += reference_counter_free(dict); - freed += sizeof(DICTIONARY) + dict->scratchpad_size; - freez(dict); + // Unfortunately, the JudyHSDel() does not return the value of the + // item that was deleted, so we have to find it before we delete it, + // since we need to release our structures too. - return freed; + dictionary_index_lock_wrlock(dict); + + int ret; + DICTIONARY_ITEM *item = hashtable_get_unsafe(dict, name, name_len); + if(unlikely(!item)) { + dictionary_index_lock_unlock(dict); + ret = false; + } + else { + if(hashtable_delete_unsafe(dict, name, name_len, item) == 0) + error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s' that is not in the index", name); + + dictionary_index_lock_unlock(dict); + + item_free_or_mark_deleted(dict, item); + ret = true; + } + + return ret; } -// ---------------------------------------------------------------------------- -// helpers - -static NAME_VALUE *dictionary_set_name_value_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len) { - if(unlikely(!name)) { - internal_error(true, "DICTIONARY: attempted to dictionary_set() a dictionary item without a name"); +static DICTIONARY_ITEM *item_add_or_reset_value_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) { + if(unlikely(!name || !*name)) { + internal_error( + true, + "DICTIONARY: attempted to %s() without a name on a dictionary created from %s() %zu@%s.", + __FUNCTION__, + dict->creation_function, + dict->creation_line, + dict->creation_file); return NULL; } - if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + if(unlikely(is_dictionary_destroyed(dict))) { internal_error(true, "DICTIONARY: attempted to dictionary_set() on a destroyed dictionary"); return NULL; } - internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: inserting dictionary item '%s' without exclusive access to dictionary", name); - - size_t name_len = strlen(name) + 1; // we need the terminating null too + if(name_len == -1) + name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name); @@ -901,269 +1357,608 @@ static NAME_VALUE *dictionary_set_name_value_unsafe(DICTIONARY *dict, const char // But the caller has the option to do this on his/her own. // So, let's do the fastest here and let the caller decide the flow of calls. - NAME_VALUE *nv, **pnv = (NAME_VALUE **)hashtable_insert_unsafe(dict, name, name_len); - if(likely(*pnv == 0)) { - // a new item added to the index - nv = *pnv = namevalue_create_unsafe(dict, name, name_len, value, value_len); - hashtable_inserted_name_value_unsafe(dict, nv); - linkedlist_namevalue_link_unsafe(dict, nv); - nv->flags |= NAME_VALUE_FLAG_NEW_OR_UPDATED; - } - else { - // the item is already in the index - // so, either we will return the old one - // or overwrite the value, depending on dictionary flags + dictionary_index_lock_wrlock(dict); - // We should not compare the values here! - // even if they are the same, we have to do the whole job - // so that the callbacks will be called. + bool added_or_updated = false; + size_t spins = 0; + DICTIONARY_ITEM *item = NULL; + do { + DICTIONARY_ITEM **item_pptr = (DICTIONARY_ITEM **)hashtable_insert_unsafe(dict, name, name_len); + if (likely(*item_pptr == 0)) { + // a new item added to the index - nv = *pnv; + // create the dictionary item + item = *item_pptr = item_create_with_hooks(dict, name, name_len, value, value_len, constructor_data, master_item); - if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE)) { - namevalue_reset_unsafe(dict, nv, value, value_len); - nv->flags |= NAME_VALUE_FLAG_NEW_OR_UPDATED; + // call the hashtable react + hashtable_inserted_item_unsafe(dict, item); + + // unlock the index lock, before we add it to the linked list + // DONT DO IT THE OTHER WAY AROUND - DO NOT CROSS THE LOCKS! + dictionary_index_lock_unlock(dict); + + item_linked_list_add(dict, item); + added_or_updated = true; } - - else if(dict->conflict_callback) { - dict->conflict_callback(namevalue_get_name(nv), nv->value, value, dict->conflict_callback_data); - nv->flags |= NAME_VALUE_FLAG_NEW_OR_UPDATED; - } - else { - // we did really nothing! - // make sure this flag is not set. - nv->flags &= ~NAME_VALUE_FLAG_NEW_OR_UPDATED; - } - } + if(item_check_and_acquire_advanced(dict, *item_pptr, true) != ITEM_OK) { + spins++; + continue; + } - return nv; + // the item is already in the index + // so, either we will return the old one + // or overwrite the value, depending on dictionary flags + + // We should not compare the values here! + // even if they are the same, we have to do the whole job + // so that the callbacks will be called. + + item = *item_pptr; + + if(is_view_dictionary(dict)) { + // view dictionary + // the item is already there and can be used + if(item->shared != master_item->shared) + error("DICTIONARY: changing the master item on a view is not supported. The previous item will remain. To change the key of an item in a view, delete it and add it again."); + } + else { + // master dictionary + // the user wants to reset its value + + if (!(dict->options & DICT_OPTION_DONT_OVERWRITE_VALUE)) { + item_reset_value_with_hooks(dict, item, value, value_len, constructor_data); + added_or_updated = true; + } + + else if (dictionary_execute_conflict_callback(dict, item, value, constructor_data)) { + dictionary_version_increment(dict); + added_or_updated = true; + } + + else { + // we did really nothing! + ; + } + } + + dictionary_index_lock_unlock(dict); + } + } while(!item); + + + if(unlikely(spins > 0 && dict->stats)) + DICTIONARY_STATS_INSERT_SPINS_PLUS(dict, spins); + + if(is_master_dictionary(dict) && added_or_updated) + dictionary_execute_react_callback(dict, item, constructor_data); + + return item; } -static NAME_VALUE *dictionary_get_name_value_unsafe(DICTIONARY *dict, const char *name) { - if(unlikely(!name)) { - internal_error(true, "attempted to dictionary_get() without a name"); +static DICTIONARY_ITEM *item_find_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len) { + if(unlikely(!name || !*name)) { + internal_error( + true, + "DICTIONARY: attempted to %s() without a name on a dictionary created from %s() %zu@%s.", + __FUNCTION__, + dict->creation_function, + dict->creation_line, + dict->creation_file); return NULL; } - if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + if(unlikely(is_dictionary_destroyed(dict))) { internal_error(true, "DICTIONARY: attempted to dictionary_get() on a destroyed dictionary"); return NULL; } - size_t name_len = strlen(name) + 1; // we need the terminating null too + if(name_len == -1) + name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name); - NAME_VALUE *nv = hashtable_get_unsafe(dict, name, name_len); - if(unlikely(!nv)) { - debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name); - return NULL; + dictionary_index_lock_rdlock(dict); + + DICTIONARY_ITEM *item = hashtable_get_unsafe(dict, name, name_len); + if(unlikely(item && !item_check_and_acquire(dict, item))) { + item = NULL; + DICTIONARY_STATS_SEARCH_IGNORES_PLUS1(dict); } - debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name); - return nv; -} + dictionary_index_lock_unlock(dict); -// ---------------------------------------------------------------------------- -// API - items management - -void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len) { - NAME_VALUE *nv = dictionary_set_name_value_unsafe(dict, name, value, value_len); - - if(unlikely(dict->react_callback && nv && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) { - // we need to call the react callback with a reference counter on nv - reference_counter_acquire(dict, nv); - dict->react_callback(namevalue_get_name(nv), nv->value, dict->react_callback_data); - reference_counter_release(dict, nv, false); - } - - return nv ? nv->value : NULL; -} - -void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) { - dictionary_lock(dict, DICTIONARY_LOCK_WRITE); - NAME_VALUE *nv = dictionary_set_name_value_unsafe(dict, name, value, value_len); - - // we need to get a reference counter for the react callback - // before we unlock the dictionary - if(unlikely(dict->react_callback && nv && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) - reference_counter_acquire(dict, nv); - - dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); - - if(unlikely(dict->react_callback && nv && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) { - // we got the reference counter we need, above - dict->react_callback(namevalue_get_name(nv), nv->value, dict->react_callback_data); - reference_counter_release(dict, nv, false); - } - - return nv ? nv->value : NULL; -} - -DICTIONARY_ITEM *dictionary_set_and_acquire_item_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len) { - NAME_VALUE *nv = dictionary_set_name_value_unsafe(dict, name, value, value_len); - - if(unlikely(!nv)) - return NULL; - - reference_counter_acquire(dict, nv); - - if(unlikely(dict->react_callback && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) { - dict->react_callback(namevalue_get_name(nv), nv->value, dict->react_callback_data); - } - - return (DICTIONARY_ITEM *)nv; -} - -DICTIONARY_ITEM *dictionary_set_and_acquire_item(DICTIONARY *dict, const char *name, void *value, size_t value_len) { - dictionary_lock(dict, DICTIONARY_LOCK_WRITE); - NAME_VALUE *nv = dictionary_set_name_value_unsafe(dict, name, value, value_len); - - // we need to get the reference counter before we unlock - if(nv) reference_counter_acquire(dict, nv); - - dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); - - if(unlikely(dict->react_callback && nv && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) { - // we already have a reference counter, for the caller, no need for another one - dict->react_callback(namevalue_get_name(nv), nv->value, dict->react_callback_data); - } - - return (DICTIONARY_ITEM *)nv; -} - -void *dictionary_get_unsafe(DICTIONARY *dict, const char *name) { - NAME_VALUE *nv = dictionary_get_name_value_unsafe(dict, name); - - if(unlikely(!nv)) - return NULL; - - return nv->value; -} - -void *dictionary_get(DICTIONARY *dict, const char *name) { - dictionary_lock(dict, DICTIONARY_LOCK_READ); - void *ret = dictionary_get_unsafe(dict, name); - dictionary_unlock(dict, DICTIONARY_LOCK_READ); - return ret; -} - -DICTIONARY_ITEM *dictionary_get_and_acquire_item_unsafe(DICTIONARY *dict, const char *name) { - NAME_VALUE *nv = dictionary_get_name_value_unsafe(dict, name); - - if(unlikely(!nv)) - return NULL; - - reference_counter_acquire(dict, nv); - return (DICTIONARY_ITEM *)nv; -} - -DICTIONARY_ITEM *dictionary_get_and_acquire_item(DICTIONARY *dict, const char *name) { - dictionary_lock(dict, DICTIONARY_LOCK_READ); - void *ret = dictionary_get_and_acquire_item_unsafe(dict, name); - dictionary_unlock(dict, DICTIONARY_LOCK_READ); - return ret; -} - -DICTIONARY_ITEM *dictionary_acquired_item_dup(DICTIONARY_ITEM *item) { - if(unlikely(!item)) return NULL; - reference_counter_increase((NAME_VALUE *)item); return item; } -const char *dictionary_acquired_item_name(DICTIONARY_ITEM *item) { - if(unlikely(!item)) return NULL; - return namevalue_get_name((NAME_VALUE *)item); -} +// ---------------------------------------------------------------------------- +// delayed destruction of dictionaries -void *dictionary_acquired_item_value(DICTIONARY_ITEM *item) { - if(unlikely(!item)) return NULL; - return ((NAME_VALUE *)item)->value; -} +static bool dictionary_free_all_resources(DICTIONARY *dict, size_t *mem, bool force) { + if(mem) + *mem = 0; -void dictionary_acquired_item_release_unsafe(DICTIONARY *dict, DICTIONARY_ITEM *item) { - if(unlikely(!item)) return; + if(!force && dictionary_referenced_items(dict)) + return false; + + size_t dict_size = 0, counted_items = 0, item_size = 0, index_size = 0; + (void)counted_items; #ifdef NETDATA_INTERNAL_CHECKS - if(((NAME_VALUE *)item)->dict != dict) - fatal("DICTIONARY: %s(): name_value item with name '%s' does not belong to this dictionary", __FUNCTION__, namevalue_get_name((NAME_VALUE *)item)); + long int entries = dict->entries; + long int referenced_items = dict->referenced_items; + long int pending_deletion_items = dict->pending_deletion_items; + const char *creation_function = dict->creation_function; + const char *creation_file = dict->creation_file; + size_t creation_line = dict->creation_line; #endif - reference_counter_release(dict, (NAME_VALUE *)item, false); + // destroy the index + index_size += hashtable_destroy_unsafe(dict); + + ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); + DICTIONARY_ITEM *item = dict->items.list; + while (item) { + // cache item->next + // because we are going to free item + DICTIONARY_ITEM *item_next = item->next; + item_size += item_free_with_hooks(dict, item); + item = item_next; + + DICTIONARY_ENTRIES_MINUS1(dict); + + // to speed up destruction, we don't + // unlink item from the linked-list here + + counted_items++; + } + dict->items.list = NULL; + ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE); + + dict_size += dictionary_locks_destroy(dict); + dict_size += reference_counter_free(dict); + dict_size += dictionary_hooks_free(dict); + dict_size += sizeof(DICTIONARY); + DICTIONARY_STATS_MINUS_MEMORY(dict, 0, sizeof(DICTIONARY), 0); + + freez(dict); + + internal_error( + true, + "DICTIONARY: Freed dictionary created from %s() %zu@%s, having %ld (counted %zu) entries, %ld referenced, %ld pending deletion, total freed memory: %zu bytes (sizeof(dict) = %zu, sizeof(item) = %zu).", + creation_function, + creation_line, + creation_file, + entries, counted_items, referenced_items, pending_deletion_items, + dict_size, sizeof(DICTIONARY), sizeof(DICTIONARY_ITEM) + sizeof(DICTIONARY_ITEM_SHARED)); + + if(mem) + *mem = dict_size + item_size + index_size; + + return true; } -void dictionary_acquired_item_release(DICTIONARY *dict, DICTIONARY_ITEM *item) { - if(unlikely(!item)) return; +netdata_mutex_t dictionaries_waiting_to_be_destroyed_mutex = NETDATA_MUTEX_INITIALIZER; +static DICTIONARY *dictionaries_waiting_to_be_destroyed = NULL; + +void dictionary_queue_for_destruction(DICTIONARY *dict) { + if(is_dictionary_destroyed(dict)) + return; + + DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(dict); + dict_flag_set(dict, DICT_FLAG_DESTROYED); + + netdata_mutex_lock(&dictionaries_waiting_to_be_destroyed_mutex); + + dict->next = dictionaries_waiting_to_be_destroyed; + dictionaries_waiting_to_be_destroyed = dict; + + netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex); +} + +void cleanup_destroyed_dictionaries(void) { + if(!dictionaries_waiting_to_be_destroyed) + return; + + netdata_mutex_lock(&dictionaries_waiting_to_be_destroyed_mutex); + + DICTIONARY *dict, *last = NULL, *next = NULL; + for(dict = dictionaries_waiting_to_be_destroyed; dict ; dict = next) { + next = dict->next; #ifdef NETDATA_INTERNAL_CHECKS - if(((NAME_VALUE *)item)->dict != dict) - fatal("DICTIONARY: %s(): name_value item with name '%s' does not belong to this dictionary", __FUNCTION__, namevalue_get_name((NAME_VALUE *)item)); + size_t line = dict->creation_line; + const char *file = dict->creation_file; + const char *function = dict->creation_function; #endif + DICTIONARY_STATS_DICT_DESTROY_QUEUED_MINUS1(dict); + if(dictionary_free_all_resources(dict, NULL, false)) { + + internal_error( + true, + "DICTIONARY: freed dictionary with delayed destruction, created from %s() %zu@%s.", + function, line, file); + + if(last) last->next = next; + else dictionaries_waiting_to_be_destroyed = next; + } + else { + DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(dict); + last = dict; + } + } + + netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex); +} + +// ---------------------------------------------------------------------------- +// API internal checks + +#ifdef NETDATA_INTERNAL_CHECKS +#define api_internal_check(dict, item, allow_null_dict, allow_null_item) api_internal_check_with_trace(dict, item, __FUNCTION__, allow_null_dict, allow_null_item) +static inline void api_internal_check_with_trace(DICTIONARY *dict, DICTIONARY_ITEM *item, const char *function, bool allow_null_dict, bool allow_null_item) { + if(!allow_null_dict && !dict) { + internal_error( + item, + "DICTIONARY: attempted to %s() with a NULL dictionary, passing an item created from %s() %zu@%s.", + function, + item->dict->creation_function, + item->dict->creation_line, + item->dict->creation_file); + fatal("DICTIONARY: attempted to %s() but item is NULL", function); + } + + if(!allow_null_item && !item) { + internal_error( + true, + "DICTIONARY: attempted to %s() without an item on a dictionary created from %s() %zu@%s.", + function, + dict?dict->creation_function:"unknown", + dict?dict->creation_line:0, + dict?dict->creation_file:"unknown"); + fatal("DICTIONARY: attempted to %s() but item is NULL", function); + } + + if(dict && item && dict != item->dict) { + internal_error( + true, + "DICTIONARY: attempted to %s() an item on a dictionary created from %s() %zu@%s, but the item belongs to the dictionary created from %s() %zu@%s.", + function, + dict->creation_function, + dict->creation_line, + dict->creation_file, + item->dict->creation_function, + item->dict->creation_line, + item->dict->creation_file + ); + fatal("DICTIONARY: %s(): item does not belong to this dictionary.", function); + } + + if(item) { + REFCOUNT refcount = DICTIONARY_ITEM_REFCOUNT_GET(dict, item); + if (unlikely(refcount <= 0)) { + internal_error( + true, + "DICTIONARY: attempted to %s() of an item with reference counter = %d on a dictionary created from %s() %zu@%s", + function, + refcount, + item->dict->creation_function, + item->dict->creation_line, + item->dict->creation_file); + fatal("DICTIONARY: attempted to %s but item is having refcount = %d", function, refcount); + } + } +} +#else +#define api_internal_check(dict, item, allow_null_dict, allow_null_item) debug_dummy() +#endif + +#define api_is_name_good(dict, name, name_len) api_is_name_good_with_trace(dict, name, name_len, __FUNCTION__) +static bool api_is_name_good_with_trace(DICTIONARY *dict __maybe_unused, const char *name, ssize_t name_len __maybe_unused, const char *function __maybe_unused) { + if(unlikely(!name)) { + internal_error( + true, + "DICTIONARY: attempted to %s() with name = NULL on a dictionary created from %s() %zu@%s.", + function, + dict?dict->creation_function:"unknown", + dict?dict->creation_line:0, + dict?dict->creation_file:"unknown"); + return false; + } + + if(unlikely(!*name)) { + internal_error( + true, + "DICTIONARY: attempted to %s() with empty name on a dictionary created from %s() %zu@%s.", + function, + dict?dict->creation_function:"unknown", + dict?dict->creation_line:0, + dict?dict->creation_file:"unknown"); + return false; + } + + internal_error( + name_len > 0 && name_len != (ssize_t)(strlen(name) + 1), + "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu (incl. '\\0'), but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.", + function, + name, + strlen(name) + 1, + name_len, + dict?dict->creation_function:"unknown", + dict?dict->creation_line:0, + dict?dict->creation_file:"unknown"); + + internal_error( + name_len <= 0 && name_len != -1, + "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu (incl. '\\0'), but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.", + function, + name, + strlen(name) + 1, + name_len, + dict?dict->creation_function:"unknown", + dict?dict->creation_line:0, + dict?dict->creation_file:"unknown"); + + return true; +} + +// ---------------------------------------------------------------------------- +// API - dictionary management + +static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dictionary_stats *stats) { + cleanup_destroyed_dictionaries(); + + DICTIONARY *dict = callocz(1, sizeof(DICTIONARY)); + dict->options = options; + dict->stats = stats; + + size_t dict_size = 0; + dict_size += sizeof(DICTIONARY); + dict_size += dictionary_locks_init(dict); + dict_size += reference_counter_init(dict); + dict_size += hashtable_init_unsafe(dict); + + DICTIONARY_STATS_PLUS_MEMORY(dict, 0, dict_size, 0); + + return dict; +} + +#ifdef NETDATA_INTERNAL_CHECKS +DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, const char *function, size_t line, const char *file) { +#else +DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats) { +#endif + + DICTIONARY *dict = dictionary_create_internal(options, stats?stats:&dictionary_stats_category_other); + +#ifdef NETDATA_INTERNAL_CHECKS + dict->creation_function = function; + dict->creation_file = file; + dict->creation_line = line; +#endif + + DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict); + return dict; +} + +#ifdef NETDATA_INTERNAL_CHECKS +DICTIONARY *dictionary_create_view_with_trace(DICTIONARY *master, const char *function, size_t line, const char *file) { +#else +DICTIONARY *dictionary_create_view(DICTIONARY *master) { +#endif + + DICTIONARY *dict = dictionary_create_internal(master->options, master->stats); + dict->master = master; + + dictionary_hooks_allocate(master); + + if(unlikely(__atomic_load_n(&master->hooks->links, __ATOMIC_SEQ_CST)) < 1) + fatal("DICTIONARY: attempted to create a view that has %d links", master->hooks->links); + + dict->hooks = master->hooks; + __atomic_add_fetch(&master->hooks->links, 1, __ATOMIC_SEQ_CST); + +#ifdef NETDATA_INTERNAL_CHECKS + dict->creation_function = function; + dict->creation_file = file; + dict->creation_line = line; +#endif + + DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict); + return dict; +} + +void dictionary_flush(DICTIONARY *dict) { + if(unlikely(!dict)) + return; + + // delete the index + dictionary_index_lock_wrlock(dict); + hashtable_destroy_unsafe(dict); + dictionary_index_lock_unlock(dict); + + // delete all items + ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); // get write lock here, to speed it up (it is recursive) + DICTIONARY_ITEM *item, *item_next; + for (item = dict->items.list; item; item = item_next) { + item_next = item->next; + + if(!item_flag_check(item, ITEM_FLAG_DELETED)) + item_free_or_mark_deleted(dict, item); + } + ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE); + + DICTIONARY_STATS_DICT_FLUSHES_PLUS1(dict); +} + +size_t dictionary_destroy(DICTIONARY *dict) { + cleanup_destroyed_dictionaries(); + + if(!dict) return 0; + + DICTIONARY_STATS_DICT_DESTRUCTIONS_PLUS1(dict); + + size_t referenced_items = dictionary_referenced_items(dict); + if(referenced_items) { + dictionary_flush(dict); + dictionary_queue_for_destruction(dict); + + internal_error( + true, + "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s, because it has %ld referenced items in it (%ld total).", + dict->creation_function, + dict->creation_line, + dict->creation_file, + dict->referenced_items, + dict->entries); + + return 0; + } + + size_t freed; + dictionary_free_all_resources(dict, &freed, true); + + return freed; +} + +// ---------------------------------------------------------------------------- +// SET an item to the dictionary + +DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_set_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data) { + if(unlikely(!api_is_name_good(dict, name, name_len))) + return NULL; + + api_internal_check(dict, NULL, false, true); + + if(unlikely(is_view_dictionary(dict))) + fatal("DICTIONARY: this dictionary is a view, you cannot add items other than the ones from the master dictionary."); + + DICTIONARY_ITEM *item = item_add_or_reset_value_and_acquire(dict, name, name_len, value, value_len, constructor_data, NULL); + api_internal_check(dict, item, false, false); + return item; +} + +void *dictionary_set_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data) { + DICTIONARY_ITEM *item = dictionary_set_and_acquire_item_advanced(dict, name, name_len, value, value_len, constructor_data); + + if(likely(item)) { + void *v = item->shared->value; + item_release(dict, item); + return v; + } + + return NULL; +} + +DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_view_set_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, DICTIONARY_ITEM *master_item) { + if(unlikely(!api_is_name_good(dict, name, name_len))) + return NULL; + + api_internal_check(dict, NULL, false, true); + + if(unlikely(is_master_dictionary(dict))) + fatal("DICTIONARY: this dictionary is a master, you cannot add items from other dictionaries."); + + dictionary_acquired_item_dup(dict->master, master_item); + DICTIONARY_ITEM *item = item_add_or_reset_value_and_acquire(dict, name, name_len, NULL, 0, NULL, master_item); + dictionary_acquired_item_release(dict->master, master_item); + + api_internal_check(dict, item, false, false); + return item; +} + +void *dictionary_view_set_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, DICTIONARY_ITEM *master_item) { + DICTIONARY_ITEM *item = dictionary_view_set_and_acquire_item_advanced(dict, name, name_len, master_item); + + if(likely(item)) { + void *v = item->shared->value; + item_release(dict, item); + return v; + } + + return NULL; +} + +// ---------------------------------------------------------------------------- +// GET an item from the dictionary + +DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_get_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len) { + if(unlikely(!api_is_name_good(dict, name, name_len))) + return NULL; + + api_internal_check(dict, NULL, false, true); + DICTIONARY_ITEM *item = item_find_and_acquire(dict, name, name_len); + api_internal_check(dict, item, false, true); + return item; +} + +void *dictionary_get_advanced(DICTIONARY *dict, const char *name, ssize_t name_len) { + DICTIONARY_ITEM *item = dictionary_get_and_acquire_item_advanced(dict, name, name_len); + + if(likely(item)) { + void *v = item->shared->value; + item_release(dict, item); + return v; + } + + return NULL; +} + +// ---------------------------------------------------------------------------- +// DUP/REL an item (increase/decrease its reference counter) + +DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_acquired_item_dup(DICTIONARY *dict, DICT_ITEM_CONST DICTIONARY_ITEM *item) { + // we allow the item to be NULL here + api_internal_check(dict, item, false, true); + + if(likely(item)) { + item_acquire(dict, item); + api_internal_check(dict, item, false, false); + } + + return item; +} + +void dictionary_acquired_item_release(DICTIONARY *dict, DICT_ITEM_CONST DICTIONARY_ITEM *item) { + // we allow the item to be NULL here + api_internal_check(dict, item, false, true); + // no need to get a lock here // we pass the last parameter to reference_counter_release() as true // so that the release may get a write-lock if required to clean up - reference_counter_release(dict, (NAME_VALUE *)item, true); - - if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) - dictionary_destroy(dict); + if(likely(item)) + item_release(dict, item); } -int dictionary_del_unsafe(DICTIONARY *dict, const char *name) { - if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { - internal_error(true, "DICTIONARY: attempted to dictionary_del() on a destroyed dictionary"); - return -1; - } +// ---------------------------------------------------------------------------- +// get the name/value of an item - if(unlikely(!name || !*name)) { - internal_error(true, "DICTIONARY: attempted to dictionary_del() without a name"); - return -1; - } - - internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: INTERNAL ERROR: deleting dictionary item '%s' without exclusive access to dictionary", name); - - size_t name_len = strlen(name) + 1; // we need the terminating null too - - debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name); - - // Unfortunately, the JudyHSDel() does not return the value of the - // item that was deleted, so we have to find it before we delete it, - // since we need to release our structures too. - - int ret; - NAME_VALUE *nv = hashtable_get_unsafe(dict, name, name_len); - if(unlikely(!nv)) { - debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name); - ret = -1; - } - else { - debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name); - - if(hashtable_delete_unsafe(dict, name, name_len, nv) == 0) - error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s' that is not in the index", name); - - if(name_value_can_be_deleted(dict, nv)) { - linkedlist_namevalue_unlink_unsafe(dict, nv); - namevalue_destroy_unsafe(dict, nv); - } - else - nv->flags |= NAME_VALUE_FLAG_DELETED; - - ret = 0; - - DICTIONARY_STATS_ENTRIES_MINUS1(dict); - - } - return ret; +const char *dictionary_acquired_item_name(DICT_ITEM_CONST DICTIONARY_ITEM *item) { + api_internal_check(NULL, item, true, false); + return item_get_name(item); } -int dictionary_del(DICTIONARY *dict, const char *name) { - dictionary_lock(dict, DICTIONARY_LOCK_WRITE); - int ret = dictionary_del_unsafe(dict, name); - dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); - return ret; +void *dictionary_acquired_item_value(DICT_ITEM_CONST DICTIONARY_ITEM *item) { + // we allow the item to be NULL here + api_internal_check(NULL, item, true, true); + + if(likely(item)) + return item->shared->value; + + return NULL; +} + +// ---------------------------------------------------------------------------- +// DEL an item + +bool dictionary_del_advanced(DICTIONARY *dict, const char *name, ssize_t name_len) { + if(unlikely(!api_is_name_good(dict, name, name_len))) + return false; + + api_internal_check(dict, NULL, false, true); + return item_del(dict, name, name_len); } // ---------------------------------------------------------------------------- @@ -1172,43 +1967,43 @@ int dictionary_del(DICTIONARY *dict, const char *name) { void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) { if(unlikely(!dfe || !dict)) return NULL; - if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + if(unlikely(is_dictionary_destroyed(dict))) { internal_error(true, "DICTIONARY: attempted to dictionary_foreach_start_rw() on a destroyed dictionary"); - dfe->last_item = NULL; + dfe->counter = 0; + dfe->item = NULL; dfe->name = NULL; dfe->value = NULL; return NULL; } + dfe->counter = 0; dfe->dict = dict; dfe->rw = rw; - dfe->started_ut = now_realtime_usec(); - dictionary_lock(dict, dfe->rw); + ll_recursive_lock(dict, dfe->rw); - DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict); + DICTIONARY_STATS_TRAVERSALS_PLUS1(dict); // get the first item from the list - NAME_VALUE *nv = dict->first_item; + DICTIONARY_ITEM *item = dict->items.list; // skip all the deleted items - while(nv && (nv->flags & NAME_VALUE_FLAG_DELETED)) - nv = nv->next; + while(item && !item_check_and_acquire(dict, item)) + item = item->next; - if(likely(nv)) { - dfe->last_item = nv; - dfe->name = (char *)namevalue_get_name(nv); - dfe->value = nv->value; - reference_counter_acquire(dict, nv); + if(likely(item)) { + dfe->item = item; + dfe->name = (char *)item_get_name(item); + dfe->value = item->shared->value; } else { - dfe->last_item = NULL; + dfe->item = NULL; dfe->name = NULL; dfe->value = NULL; } if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) - dictionary_unlock(dfe->dict, dfe->rw); + ll_recursive_unlock(dfe->dict, dfe->rw); return dfe->value; } @@ -1216,123 +2011,121 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) { void *dictionary_foreach_next(DICTFE *dfe) { if(unlikely(!dfe || !dfe->dict)) return NULL; - if(unlikely(dfe->dict->flags & DICTIONARY_FLAG_DESTROYED)) { + if(unlikely(is_dictionary_destroyed(dfe->dict))) { internal_error(true, "DICTIONARY: attempted to dictionary_foreach_next() on a destroyed dictionary"); - dfe->last_item = NULL; + dfe->item = NULL; dfe->name = NULL; dfe->value = NULL; return NULL; } if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) - dictionary_lock(dfe->dict, dfe->rw); + ll_recursive_lock(dfe->dict, dfe->rw); // the item we just did - NAME_VALUE *nv = (NAME_VALUE *)dfe->last_item; + DICTIONARY_ITEM *item = dfe->item; // get the next item from the list - NAME_VALUE *nv_next = (nv) ? nv->next : NULL; + DICTIONARY_ITEM *item_next = (item) ? item->next : NULL; - // skip all the deleted items - while(nv_next && (nv_next->flags & NAME_VALUE_FLAG_DELETED)) - nv_next = nv_next->next; + // skip all the deleted items until one that can be acquired is found + while(item_next && !item_check_and_acquire(dfe->dict, item_next)) + item_next = item_next->next; - // release the old, so that it can possibly be deleted - if(likely(nv)) - reference_counter_release(dfe->dict, nv, false); + if(likely(item)) { + item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dfe->dict, item, dfe->rw); + // item_release(dfe->dict, item); + } - if(likely(nv = nv_next)) { - dfe->last_item = nv; - dfe->name = (char *)namevalue_get_name(nv); - dfe->value = nv->value; - reference_counter_acquire(dfe->dict, nv); + item = item_next; + if(likely(item)) { + dfe->item = item; + dfe->name = (char *)item_get_name(item); + dfe->value = item->shared->value; + dfe->counter++; } else { - dfe->last_item = NULL; + dfe->item = NULL; dfe->name = NULL; dfe->value = NULL; } if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) - dictionary_unlock(dfe->dict, dfe->rw); + ll_recursive_unlock(dfe->dict, dfe->rw); return dfe->value; } -usec_t dictionary_foreach_done(DICTFE *dfe) { - if(unlikely(!dfe || !dfe->dict)) return 0; +void dictionary_foreach_done(DICTFE *dfe) { + if(unlikely(!dfe || !dfe->dict)) return; - if(unlikely(dfe->dict->flags & DICTIONARY_FLAG_DESTROYED)) { + if(unlikely(is_dictionary_destroyed(dfe->dict))) { internal_error(true, "DICTIONARY: attempted to dictionary_foreach_next() on a destroyed dictionary"); - return 0; + return; } // the item we just did - NAME_VALUE *nv = (NAME_VALUE *)dfe->last_item; + DICTIONARY_ITEM *item = dfe->item; // release it, so that it can possibly be deleted - if(likely(nv)) - reference_counter_release(dfe->dict, nv, false); + if(likely(item)) { + item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dfe->dict, item, dfe->rw); + // item_release(dfe->dict, item); + } if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT)) - dictionary_unlock(dfe->dict, dfe->rw); + ll_recursive_unlock(dfe->dict, dfe->rw); dfe->dict = NULL; - dfe->last_item = NULL; + dfe->item = NULL; dfe->name = NULL; dfe->value = NULL; - - usec_t usec = now_realtime_usec() - dfe->started_ut; - dfe->started_ut = 0; - - return usec; + dfe->counter = 0; } // ---------------------------------------------------------------------------- -// API - walk through the dictionary -// the dictionary is locked for reading while this happens +// API - walk through the dictionary. +// The dictionary is locked for reading while this happens // do not use other dictionary calls while walking the dictionary - deadlock! -int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) { - if(unlikely(!dict)) return 0; +int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data) { + if(unlikely(!dict || !callback)) return 0; - if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + if(unlikely(is_dictionary_destroyed(dict))) { internal_error(true, "DICTIONARY: attempted to dictionary_walkthrough_rw() on a destroyed dictionary"); return 0; } - dictionary_lock(dict, rw); + ll_recursive_lock(dict, rw); DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict); // written in such a way, that the callback can delete the active element int ret = 0; - NAME_VALUE *nv = dict->first_item, *nv_next; - while(nv) { + DICTIONARY_ITEM *item = dict->items.list, *item_next; + while(item) { // skip the deleted items - if(unlikely(nv->flags & NAME_VALUE_FLAG_DELETED)) { - nv = nv->next; + if(unlikely(!item_check_and_acquire(dict, item))) { + item = item->next; continue; } - // get a reference counter, so that our item will not be deleted - // while we are using it - reference_counter_acquire(dict, nv); + if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) + ll_recursive_unlock(dict, rw); + + int r = callback(item, item->shared->value, data); if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) - dictionary_unlock(dict, rw); - - int r = callback(namevalue_get_name(nv), nv->value, data); - - if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) - dictionary_lock(dict, rw); + ll_recursive_lock(dict, rw); // since we have a reference counter, this item cannot be deleted // until we release the reference counter, so the pointers are there - nv_next = nv->next; - reference_counter_release(dict, nv, false); + item_next = item->next; + + item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dict, item, rw); + // item_release(dict, item); if(unlikely(r < 0)) { ret = r; @@ -1341,10 +2134,10 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const c ret += r; - nv = nv_next; + item = item_next; } - dictionary_unlock(dict, rw); + ll_recursive_unlock(dict, rw); return ret; } @@ -1352,409 +2145,65 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const c // ---------------------------------------------------------------------------- // sorted walkthrough -static int dictionary_sort_compar(const void *nv1, const void *nv2) { - return strcmp(namevalue_get_name((*(NAME_VALUE **)nv1)), namevalue_get_name((*(NAME_VALUE **)nv2))); +static int dictionary_sort_compar(const void *item1, const void *item2) { + return strcmp(item_get_name((*(DICTIONARY_ITEM **)item1)), item_get_name((*(DICTIONARY_ITEM **)item2))); } -int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) { - if(unlikely(!dict || !dict->entries)) return 0; +int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data) { + if(unlikely(!dict || !callback)) return 0; - if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + if(unlikely(is_dictionary_destroyed(dict))) { internal_error(true, "DICTIONARY: attempted to dictionary_sorted_walkthrough_rw() on a destroyed dictionary"); return 0; } - dictionary_lock(dict, rw); - dictionary_defer_all_deletions_unsafe(dict, rw); - DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict); - size_t count = dict->entries; - NAME_VALUE **array = mallocz(sizeof(NAME_VALUE *) * count); + ll_recursive_lock(dict, rw); + size_t entries = __atomic_load_n(&dict->entries, __ATOMIC_SEQ_CST); + DICTIONARY_ITEM **array = mallocz(sizeof(DICTIONARY_ITEM *) * entries); size_t i; - NAME_VALUE *nv; - for(nv = dict->first_item, i = 0; nv && i < count ;nv = nv->next) { - if(likely(!(nv->flags & NAME_VALUE_FLAG_DELETED))) - array[i++] = nv; + DICTIONARY_ITEM *item; + for(item = dict->items.list, i = 0; item && i < entries; item = item->next) { + if(likely(item_check_and_acquire(dict, item))) + array[i++] = item; } + ll_recursive_unlock(dict, rw); - internal_error(nv != NULL, "DICTIONARY: during sorting expected to have %zu items in dictionary, but there are more. Sorted results may be incomplete. Dictionary fails to maintain an accurate number of the number of entries it has.", count); + if(unlikely(i != entries)) + entries = i; - if(unlikely(i != count)) { - internal_error(true, "DICTIONARY: during sorting expected to have %zu items in dictionary, but there are %zu. Sorted results may be incomplete. Dictionary fails to maintain an accurate number of the number of entries it has.", count, i); - count = i; - } + qsort(array, entries, sizeof(DICTIONARY_ITEM *), dictionary_sort_compar); - qsort(array, count, sizeof(NAME_VALUE *), dictionary_sort_compar); + bool callit = true; + int ret = 0, r; + for(i = 0; i < entries ;i++) { + item = array[i]; - int ret = 0; - for(i = 0; i < count ;i++) { - nv = array[i]; - if(likely(!(nv->flags & NAME_VALUE_FLAG_DELETED))) { - reference_counter_acquire(dict, nv); + if(callit) + r = callback(item, item->shared->value, data); - if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) - dictionary_unlock(dict, rw); + item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dict, item, rw); + // item_release(dict, item); - int r = callback(namevalue_get_name(nv), nv->value, data); + if(r < 0) { + ret = r; + r = 0; - if(unlikely(rw == DICTIONARY_LOCK_REENTRANT)) - dictionary_lock(dict, rw); - - reference_counter_release(dict, nv, false); - if (r < 0) { - ret = r; - break; - } - ret += r; + // stop calling the callback, + // but we have to continue, to release all the reference counters + callit = false; } + else + ret += r; } - dictionary_restore_all_deletions_unsafe(dict, rw); - dictionary_unlock(dict, rw); freez(array); return ret; } -// ---------------------------------------------------------------------------- -// STRING implementation - dedup all STRINGs - -struct netdata_string { - uint32_t length; // the string length including the terminating '\0' - - int32_t refcount; // how many times this string is used - // We use a signed number to be able to detect duplicate frees of a string. - // If at any point this goes below zero, we have a duplicate free. - - const char str[]; // the string itself, is appended to this structure -}; - -static struct string_hashtable { - Pvoid_t JudyHSArray; // the Judy array - hashtable - netdata_rwlock_t rwlock; // the R/W lock to protect the Judy array - - long int entries; // the number of entries in the index - long int active_references; // the number of active references alive - long int memory; // the memory used, without the JudyHS index - - size_t inserts; // the number of successful inserts to the index - size_t deletes; // the number of successful deleted from the index - size_t searches; // the number of successful searches in the index - size_t duplications; // when a string is referenced - size_t releases; // when a string is unreferenced - -#ifdef NETDATA_INTERNAL_CHECKS - // internal statistics - size_t found_deleted_on_search; - size_t found_available_on_search; - size_t found_deleted_on_insert; - size_t found_available_on_insert; - size_t spins; -#endif - -} string_base = { - .JudyHSArray = NULL, - .rwlock = NETDATA_RWLOCK_INITIALIZER, -}; - -#ifdef NETDATA_INTERNAL_CHECKS -#define string_internal_stats_add(var, val) __atomic_add_fetch(&string_base.var, val, __ATOMIC_RELAXED) -#else -#define string_internal_stats_add(var, val) do {;} while(0) -#endif - -#define string_stats_atomic_increment(var) __atomic_add_fetch(&string_base.var, 1, __ATOMIC_RELAXED) -#define string_stats_atomic_decrement(var) __atomic_sub_fetch(&string_base.var, 1, __ATOMIC_RELAXED) - -void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) { - *inserts = string_base.inserts; - *deletes = string_base.deletes; - *searches = string_base.searches; - *entries = (size_t)string_base.entries; - *references = (size_t)string_base.active_references; - *memory = (size_t)string_base.memory; - *duplications = string_base.duplications; - *releases = string_base.releases; -} - -#define string_entry_acquire(se) __atomic_add_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); -#define string_entry_release(se) __atomic_sub_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); - -static inline bool string_entry_check_and_acquire(STRING *se) { - int32_t expected, desired, count = 0; - do { - count++; - - expected = se->refcount; - - if(expected <= 0) { - // We cannot use this. - // The reference counter reached value zero, - // so another thread is deleting this. - string_internal_stats_add(spins, count - 1); - return false; - } - - desired = expected + 1; - } - while(!__atomic_compare_exchange_n(&se->refcount, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)); - - string_internal_stats_add(spins, count - 1); - - // statistics - // string_base.active_references is altered at the in string_strdupz() and string_freez() - string_stats_atomic_increment(duplications); - - return true; -} - -STRING *string_dup(STRING *string) { - if(unlikely(!string)) return NULL; - -#ifdef NETDATA_INTERNAL_CHECKS - if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) <= 0)) - fatal("STRING: tried to %s() a string that is freed (it has %d references).", __FUNCTION__, string->refcount); -#endif - - string_entry_acquire(string); - - // statistics - string_stats_atomic_increment(active_references); - string_stats_atomic_increment(duplications); - - return string; -} - -// Search the index and return an ACQUIRED string entry, or NULL -static inline STRING *string_index_search(const char *str, size_t length) { - if(unlikely(!string_base.JudyHSArray)) - return NULL; - - STRING *string; - - // Find the string in the index - // With a read-lock so that multiple readers can use the index concurrently. - - netdata_rwlock_rdlock(&string_base.rwlock); - - Pvoid_t *Rc; - Rc = JudyHSGet(string_base.JudyHSArray, (void *)str, length); - if(likely(Rc)) { - // found in the hash table - string = *Rc; - - if(string_entry_check_and_acquire(string)) { - // we can use this entry - string_internal_stats_add(found_available_on_search, 1); - } - else { - // this entry is about to be deleted by another thread - // do not touch it, let it go... - string = NULL; - string_internal_stats_add(found_deleted_on_search, 1); - } - } - else { - // not found in the hash table - string = NULL; - } - - string_stats_atomic_increment(searches); - netdata_rwlock_unlock(&string_base.rwlock); - - return string; -} - -// Insert a string to the index and return an ACQUIRED string entry, -// or NULL if the call needs to be retried (a deleted entry with the same key is still in the index) -// The returned entry is ACQUIRED and it can either be: -// 1. a new item inserted, or -// 2. an item found in the index that is not currently deleted -static inline STRING *string_index_insert(const char *str, size_t length) { - STRING *string; - - netdata_rwlock_wrlock(&string_base.rwlock); - - STRING **ptr; - { - JError_t J_Error; - Pvoid_t *Rc = JudyHSIns(&string_base.JudyHSArray, (void *)str, length, &J_Error); - if (unlikely(Rc == PJERR)) { - fatal( - "STRING: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d", - str, - JU_ERRNO(&J_Error), - JU_ERRID(&J_Error)); - } - ptr = (STRING **)Rc; - } - - if (likely(*ptr == 0)) { - // a new item added to the index - size_t mem_size = sizeof(STRING) + length; - string = mallocz(mem_size); - strcpy((char *)string->str, str); - string->length = length; - string->refcount = 1; - *ptr = string; - string_base.inserts++; - string_base.entries++; - string_base.memory += (long)mem_size; - } - else { - // the item is already in the index - string = *ptr; - - if(string_entry_check_and_acquire(string)) { - // we can use this entry - string_internal_stats_add(found_available_on_insert, 1); - } - else { - // this entry is about to be deleted by another thread - // do not touch it, let it go... - string = NULL; - string_internal_stats_add(found_deleted_on_insert, 1); - } - - string_stats_atomic_increment(searches); - } - - netdata_rwlock_unlock(&string_base.rwlock); - return string; -} - -// delete an entry from the index -static inline void string_index_delete(STRING *string) { - netdata_rwlock_wrlock(&string_base.rwlock); - -#ifdef NETDATA_INTERNAL_CHECKS - if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) != 0)) - fatal("STRING: tried to delete a string at %s() that is already freed (it has %d references).", __FUNCTION__, string->refcount); -#endif - - bool deleted = false; - - if (likely(string_base.JudyHSArray)) { - JError_t J_Error; - int ret = JudyHSDel(&string_base.JudyHSArray, (void *)string->str, string->length, &J_Error); - if (unlikely(ret == JERR)) { - error( - "STRING: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d", - string->str, - JU_ERRNO(&J_Error), - JU_ERRID(&J_Error)); - } else - deleted = true; - } - - if (unlikely(!deleted)) - error("STRING: tried to delete '%s' that is not in the index. Ignoring it.", string->str); - else { - size_t mem_size = sizeof(STRING) + string->length; - string_base.deletes++; - string_base.entries--; - string_base.memory -= (long)mem_size; - freez(string); - } - - netdata_rwlock_unlock(&string_base.rwlock); -} - -STRING *string_strdupz(const char *str) { - if(unlikely(!str || !*str)) return NULL; - - size_t length = strlen(str) + 1; - STRING *string = string_index_search(str, length); - - while(!string) { - // The search above did not find anything, - // We loop here, because during insert we may find an entry that is being deleted by another thread. - // So, we have to let it go and retry to insert it again. - - string = string_index_insert(str, length); - } - - // statistics - string_stats_atomic_increment(active_references); - - return string; -} - -void string_freez(STRING *string) { - if(unlikely(!string)) return; - - int32_t refcount = string_entry_release(string); - -#ifdef NETDATA_INTERNAL_CHECKS - if(unlikely(refcount < 0)) - fatal("STRING: tried to %s() a string that is already freed (it has %d references).", __FUNCTION__, string->refcount); -#endif - - if(unlikely(refcount == 0)) - string_index_delete(string); - - // statistics - string_stats_atomic_decrement(active_references); - string_stats_atomic_increment(releases); -} - -size_t string_strlen(STRING *string) { - if(unlikely(!string)) return 0; - return string->length - 1; -} - -const char *string2str(STRING *string) { - if(unlikely(!string)) return ""; - return string->str; -} - -STRING *string_2way_merge(STRING *a, STRING *b) { - static STRING *X = NULL; - - if(unlikely(!X)) { - X = string_strdupz("[x]"); - } - - if(unlikely(a == b)) return string_dup(a); - if(unlikely(a == X)) return string_dup(a); - if(unlikely(b == X)) return string_dup(b); - if(unlikely(!a)) return string_dup(X); - if(unlikely(!b)) return string_dup(X); - - size_t alen = string_strlen(a); - size_t blen = string_strlen(b); - size_t length = alen + blen + string_strlen(X) + 1; - char buf1[length + 1], buf2[length + 1], *dst1; - const char *s1, *s2; - - s1 = string2str(a); - s2 = string2str(b); - dst1 = buf1; - for( ; *s1 && *s2 && *s1 == *s2 ;s1++, s2++) - *dst1++ = *s1; - - *dst1 = '\0'; - - if(*s1 != '\0' || *s2 != '\0') { - *dst1++ = '['; - *dst1++ = 'x'; - *dst1++ = ']'; - - s1 = &(string2str(a))[alen - 1]; - s2 = &(string2str(b))[blen - 1]; - char *dst2 = &buf2[length]; - *dst2 = '\0'; - for (; *s1 && *s2 && *s1 == *s2; s1--, s2--) - *(--dst2) = *s1; - - strcpy(dst1, dst2); - } - - return string_strdupz(buf1); -} - // ---------------------------------------------------------------------------- // THREAD_CACHE @@ -1845,12 +2294,12 @@ static size_t dictionary_unittest_set_clone(DICTIONARY *dict, char **names, char static size_t dictionary_unittest_set_null(DICTIONARY *dict, char **names, char **values, size_t entries) { (void)values; size_t errors = 0; - long i = 0; - for(; i < (long)entries ;i++) { + size_t i = 0; + for(; i < entries ;i++) { void *val = dictionary_set(dict, names[i], NULL, 0); if(val != NULL) { fprintf(stderr, ">>> %s() returns a non NULL value\n", __FUNCTION__); errors++; } } - if(dictionary_stats_entries(dict) != i) { + if(dictionary_entries(dict) != i) { fprintf(stderr, ">>> %s() dictionary items do not match\n", __FUNCTION__); errors++; } @@ -1902,8 +2351,8 @@ static size_t dictionary_unittest_del_nonexisting(DICTIONARY *dict, char **names (void)names; size_t errors = 0; for(size_t i = 0; i < entries ;i++) { - int ret = dictionary_del(dict, values[i]); - if(ret != -1) { fprintf(stderr, ">>> %s() deleted non-existing item\n", __FUNCTION__); errors++; } + bool ret = dictionary_del(dict, values[i]); + if(ret) { fprintf(stderr, ">>> %s() deleted non-existing item\n", __FUNCTION__); errors++; } } return errors; } @@ -1917,18 +2366,18 @@ static size_t dictionary_unittest_del_existing(DICTIONARY *dict, char **names, c size_t backward_from = middle_to, backward_to = entries; for(size_t i = forward_from; i < forward_to ;i++) { - int ret = dictionary_del(dict, names[i]); - if(ret == -1) { fprintf(stderr, ">>> %s() didn't delete (forward) existing item\n", __FUNCTION__); errors++; } + bool ret = dictionary_del(dict, names[i]); + if(!ret) { fprintf(stderr, ">>> %s() didn't delete (forward) existing item\n", __FUNCTION__); errors++; } } for(size_t i = middle_to - 1; i >= middle_from ;i--) { - int ret = dictionary_del(dict, names[i]); - if(ret == -1) { fprintf(stderr, ">>> %s() didn't delete (middle) existing item\n", __FUNCTION__); errors++; } + bool ret = dictionary_del(dict, names[i]); + if(!ret) { fprintf(stderr, ">>> %s() didn't delete (middle) existing item\n", __FUNCTION__); errors++; } } for(size_t i = backward_to - 1; i >= backward_from ;i--) { - int ret = dictionary_del(dict, names[i]); - if(ret == -1) { fprintf(stderr, ">>> %s() didn't delete (backward) existing item\n", __FUNCTION__); errors++; } + bool ret = dictionary_del(dict, names[i]); + if(!ret) { fprintf(stderr, ">>> %s() didn't delete (backward) existing item\n", __FUNCTION__); errors++; } } return errors; @@ -1971,10 +2420,7 @@ static size_t dictionary_unittest_reset_dont_overwrite_nonclone(DICTIONARY *dict return errors; } -static int dictionary_unittest_walkthrough_callback(const char *name, void *value, void *data) { - (void)name; - (void)value; - (void)data; +static int dictionary_unittest_walkthrough_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *data __maybe_unused) { return 1; } @@ -1986,10 +2432,10 @@ static size_t dictionary_unittest_walkthrough(DICTIONARY *dict, char **names, ch else return sum - entries; } -static int dictionary_unittest_walkthrough_delete_this_callback(const char *name, void *value, void *data) { - (void)value; +static int dictionary_unittest_walkthrough_delete_this_callback(const DICTIONARY_ITEM *item, void *value __maybe_unused, void *data) { + const char *name = dictionary_acquired_item_name((DICTIONARY_ITEM *)item); - if(dictionary_del_having_write_lock((DICTIONARY *)data, name) == -1) + if(!dictionary_del((DICTIONARY *)data, name)) return 0; return 1; @@ -2003,10 +2449,7 @@ static size_t dictionary_unittest_walkthrough_delete_this(DICTIONARY *dict, char else return sum - entries; } -static int dictionary_unittest_walkthrough_stop_callback(const char *name, void *value, void *data) { - (void)name; - (void)value; - (void)data; +static int dictionary_unittest_walkthrough_stop_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *data __maybe_unused) { return -1; } @@ -2040,7 +2483,7 @@ static size_t dictionary_unittest_foreach_delete_this(DICTIONARY *dict, char **n size_t count = 0; char *item; dfe_start_write(dict, item) - if(dictionary_del_having_write_lock(dict, item_name) != -1) count++; + if(dictionary_del(dict, item_dfe.name)) count++; dfe_done(item); if(count > entries) return count - entries; @@ -2066,7 +2509,22 @@ static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *m if(callback == dictionary_unittest_destroy) dict = NULL; - fprintf(stderr, " %zu errors, %ld items in dictionary, %llu usec \n", errs, dict? dictionary_stats_entries(dict):0, dt); + long int found_ok = 0, found_deleted = 0, found_referenced = 0; + if(dict) { + DICTIONARY_ITEM *item; + DOUBLE_LINKED_LIST_FOREACH_FORWARD(dict->items.list, item, prev, next) { + if(item->refcount >= 0 && !(item ->flags & ITEM_FLAG_DELETED)) + found_ok++; + else + found_deleted++; + + if(item->refcount > 0) + found_referenced++; + } + } + + fprintf(stderr, " %zu errors, %ld (found %ld) items in dictionary, %ld (found %ld) referenced, %ld (found %ld) deleted, %llu usec \n", + errs, dict?dict->entries:0, found_ok, dict?dict->referenced_items:0, found_referenced, dict?dict->pending_deletion_items:0, found_deleted, dt); *errors += errs; return dt; } @@ -2102,23 +2560,24 @@ static void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char ** } struct dictionary_unittest_sorting { - const char *oldname; - const char *oldvalue; + const char *old_name; + const char *old_value; size_t count; }; -static int dictionary_unittest_sorting_callback(const char *name, void *value, void *data) { +static int dictionary_unittest_sorting_callback(const DICTIONARY_ITEM *item, void *value, void *data) { + const char *name = dictionary_acquired_item_name((DICTIONARY_ITEM *)item); struct dictionary_unittest_sorting *t = (struct dictionary_unittest_sorting *)data; const char *v = (const char *)value; int ret = 0; - if(t->oldname && strcmp(t->oldname, name) > 0) { - fprintf(stderr, "name '%s' should be after '%s'\n", t->oldname, name); + if(t->old_name && strcmp(t->old_name, name) > 0) { + fprintf(stderr, "name '%s' should be after '%s'\n", t->old_name, name); ret = 1; } t->count++; - t->oldname = name; - t->oldvalue = v; + t->old_name = name; + t->old_value = v; return ret; } @@ -2126,7 +2585,7 @@ static int dictionary_unittest_sorting_callback(const char *name, void *value, v static size_t dictionary_unittest_sorted_walkthrough(DICTIONARY *dict, char **names, char **values, size_t entries) { (void)names; (void)values; - struct dictionary_unittest_sorting tmp = { .oldname = NULL, .oldvalue = NULL, .count = 0 }; + struct dictionary_unittest_sorting tmp = { .old_name = NULL, .old_value = NULL, .count = 0 }; size_t errors; errors = dictionary_sorted_walkthrough_read(dict, dictionary_unittest_sorting_callback, &tmp); @@ -2148,62 +2607,94 @@ static void dictionary_unittest_null_dfe(DICTIONARY *dict, char **names, char ** } -static int check_dictionary_callback(const char *name, void *value, void *data) { - (void)name; - (void)value; - (void)data; +static int unittest_check_dictionary_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *data __maybe_unused) { return 1; } -static size_t check_dictionary(DICTIONARY *dict, size_t entries, size_t linked_list_members) { +static size_t unittest_check_dictionary(const char *label, DICTIONARY *dict, size_t traversable, size_t active_items, size_t deleted_items, size_t referenced_items, size_t pending_deletion) { size_t errors = 0; - fprintf(stderr, "dictionary entries %ld, expected %zu...\t\t\t\t\t", dictionary_stats_entries(dict), entries); - if (dictionary_stats_entries(dict) != (long)entries) { - fprintf(stderr, "FAILED\n"); - errors++; - } - else - fprintf(stderr, "OK\n"); - size_t ll = 0; void *t; dfe_start_read(dict, t) ll++; dfe_done(t); - fprintf(stderr, "dictionary foreach entries %zu, expected %zu...\t\t\t\t", ll, entries); - if(ll != entries) { + fprintf(stderr, "DICT %-20s: dictionary foreach entries %zu, expected %zu...\t\t\t\t\t", + label, ll, traversable); + if(ll != traversable) { fprintf(stderr, "FAILED\n"); errors++; } else fprintf(stderr, "OK\n"); - ll = dictionary_walkthrough_read(dict, check_dictionary_callback, NULL); - fprintf(stderr, "dictionary walkthrough entries %zu, expected %zu...\t\t\t\t", ll, entries); - if(ll != entries) { + ll = dictionary_walkthrough_read(dict, unittest_check_dictionary_callback, NULL); + fprintf(stderr, "DICT %-20s: dictionary walkthrough entries %zu, expected %zu...\t\t\t\t", + label, ll, traversable); + if(ll != traversable) { fprintf(stderr, "FAILED\n"); errors++; } else fprintf(stderr, "OK\n"); - ll = dictionary_sorted_walkthrough_read(dict, check_dictionary_callback, NULL); - fprintf(stderr, "dictionary sorted walkthrough entries %zu, expected %zu...\t\t\t", ll, entries); - if(ll != entries) { + ll = dictionary_sorted_walkthrough_read(dict, unittest_check_dictionary_callback, NULL); + fprintf(stderr, "DICT %-20s: dictionary sorted walkthrough entries %zu, expected %zu...\t\t\t", + label, ll, traversable); + if(ll != traversable) { fprintf(stderr, "FAILED\n"); errors++; } else fprintf(stderr, "OK\n"); - NAME_VALUE *nv; - for(ll = 0, nv = dict->first_item; nv ;nv = nv->next) - ll++; + DICTIONARY_ITEM *item; + size_t active = 0, deleted = 0, referenced = 0, pending = 0; + for(item = dict->items.list; item; item = item->next) { + if(!(item->flags & ITEM_FLAG_DELETED) && !(item->shared->flags & ITEM_FLAG_DELETED)) + active++; + else { + deleted++; - fprintf(stderr, "dictionary linked list entries %zu, expected %zu...\t\t\t\t", ll, linked_list_members); - if(ll != linked_list_members) { + if(item->refcount == 0) + pending++; + } + + if(item->refcount > 0) + referenced++; + } + + fprintf(stderr, "DICT %-20s: dictionary active items reported %ld, counted %zu, expected %zu...\t\t\t", + label, dict->entries, active, active_items); + if(active != active_items || active != (size_t)dict->entries) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + fprintf(stderr, "DICT %-20s: dictionary deleted items counted %zu, expected %zu...\t\t\t\t", + label, deleted, deleted_items); + if(deleted != deleted_items) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + fprintf(stderr, "DICT %-20s: dictionary referenced items reported %ld, counted %zu, expected %zu...\t\t", + label, dict->referenced_items, referenced, referenced_items); + if(referenced != referenced_items || dict->referenced_items != (long int)referenced) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + fprintf(stderr, "DICT %-20s: dictionary pending deletion items reported %ld, counted %zu, expected %zu...\t", + label, dict->pending_deletion_items, pending, pending_deletion); + if(pending != pending_deletion || pending != (size_t)dict->pending_deletion_items) { fprintf(stderr, "FAILED\n"); errors++; } @@ -2213,40 +2704,44 @@ static size_t check_dictionary(DICTIONARY *dict, size_t entries, size_t linked_l return errors; } -static int check_name_value_callback(const char *name, void *value, void *data) { - (void)name; +static int check_item_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { return value == data; } -static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, const char *name, const char *value, unsigned refcount, NAME_VALUE_FLAGS deleted_flags, bool searchable, bool browsable, bool linked) { +static size_t unittest_check_item(const char *label, DICTIONARY *dict, + DICTIONARY_ITEM *item, const char *name, const char *value, int refcount, + ITEM_FLAGS deleted_flags, bool searchable, bool browsable, bool linked) { size_t errors = 0; - fprintf(stderr, "NAME_VALUE name is '%s', expected '%s'...\t\t\t\t", namevalue_get_name(nv), name); - if(strcmp(namevalue_get_name(nv), name) != 0) { + fprintf(stderr, "ITEM %-20s: name is '%s', expected '%s'...\t\t\t\t\t\t", label, item_get_name(item), name); + if(strcmp(item_get_name(item), name) != 0) { fprintf(stderr, "FAILED\n"); errors++; } else fprintf(stderr, "OK\n"); - fprintf(stderr, "NAME_VALUE value is '%s', expected '%s'...\t\t\t", (const char *)nv->value, value); - if(strcmp((const char *)nv->value, value) != 0) { + fprintf(stderr, "ITEM %-20s: value is '%s', expected '%s'...\t\t\t\t\t", label, (const char *)item->shared->value, value); + if(strcmp((const char *)item->shared->value, value) != 0) { fprintf(stderr, "FAILED\n"); errors++; } else fprintf(stderr, "OK\n"); - fprintf(stderr, "NAME_VALUE refcount is %u, expected %u...\t\t\t\t\t", nv->refcount, refcount); - if (nv->refcount != refcount) { + fprintf(stderr, "ITEM %-20s: refcount is %d, expected %d...\t\t\t\t\t\t\t", label, item->refcount, refcount); + if (item->refcount != refcount) { fprintf(stderr, "FAILED\n"); errors++; } else fprintf(stderr, "OK\n"); - fprintf(stderr, "NAME_VALUE deleted flag is %s, expected %s...\t\t\t", (nv->flags & NAME_VALUE_FLAG_DELETED)?"TRUE":"FALSE", (deleted_flags & NAME_VALUE_FLAG_DELETED)?"TRUE":"FALSE"); - if ((nv->flags & NAME_VALUE_FLAG_DELETED) != (deleted_flags & NAME_VALUE_FLAG_DELETED)) { + fprintf(stderr, "ITEM %-20s: deleted flag is %s, expected %s...\t\t\t\t\t", label, + (item->flags & ITEM_FLAG_DELETED || item->shared->flags & ITEM_FLAG_DELETED)?"true":"false", + (deleted_flags & ITEM_FLAG_DELETED)?"true":"false"); + + if ((item->flags & ITEM_FLAG_DELETED || item->shared->flags & ITEM_FLAG_DELETED) != (deleted_flags & ITEM_FLAG_DELETED)) { fprintf(stderr, "FAILED\n"); errors++; } @@ -2254,8 +2749,9 @@ static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, co fprintf(stderr, "OK\n"); void *v = dictionary_get(dict, name); - bool found = v == nv->value; - fprintf(stderr, "NAME_VALUE searchable %5s, expected %5s...\t\t\t\t", found?"true":"false", searchable?"true":"false"); + bool found = v == item->shared->value; + fprintf(stderr, "ITEM %-20s: searchable %5s, expected %5s...\t\t\t\t\t\t", label, + found?"true":"false", searchable?"true":"false"); if(found != searchable) { fprintf(stderr, "FAILED\n"); errors++; @@ -2266,11 +2762,12 @@ static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, co found = false; void *t; dfe_start_read(dict, t) { - if(t == nv->value) found = true; + if(t == item->shared->value) found = true; } dfe_done(t); - fprintf(stderr, "NAME_VALUE dfe browsable %5s, expected %5s...\t\t\t", found?"true":"false", browsable?"true":"false"); + fprintf(stderr, "ITEM %-20s: dfe browsable %5s, expected %5s...\t\t\t\t\t", label, + found?"true":"false", browsable?"true":"false"); if(found != browsable) { fprintf(stderr, "FAILED\n"); errors++; @@ -2278,8 +2775,9 @@ static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, co else fprintf(stderr, "OK\n"); - found = dictionary_walkthrough_read(dict, check_name_value_callback, nv->value); - fprintf(stderr, "NAME_VALUE walkthrough browsable %5s, expected %5s...\t\t", found?"true":"false", browsable?"true":"false"); + found = dictionary_walkthrough_read(dict, check_item_callback, item->shared->value); + fprintf(stderr, "ITEM %-20s: walkthrough browsable %5s, expected %5s...\t\t\t\t", label, + found?"true":"false", browsable?"true":"false"); if(found != browsable) { fprintf(stderr, "FAILED\n"); errors++; @@ -2287,8 +2785,9 @@ static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, co else fprintf(stderr, "OK\n"); - found = dictionary_sorted_walkthrough_read(dict, check_name_value_callback, nv->value); - fprintf(stderr, "NAME_VALUE sorted walkthrough browsable %5s, expected %5s...\t", found?"true":"false", browsable?"true":"false"); + found = dictionary_sorted_walkthrough_read(dict, check_item_callback, item->shared->value); + fprintf(stderr, "ITEM %-20s: sorted walkthrough browsable %5s, expected %5s...\t\t\t", label, + found?"true":"false", browsable?"true":"false"); if(found != browsable) { fprintf(stderr, "FAILED\n"); errors++; @@ -2297,11 +2796,12 @@ static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, co fprintf(stderr, "OK\n"); found = false; - NAME_VALUE *n; - for(n = dict->first_item; n ;n = n->next) - if(n == nv) found = true; + DICTIONARY_ITEM *n; + for(n = dict->items.list; n ;n = n->next) + if(n == item) found = true; - fprintf(stderr, "NAME_VALUE linked %5s, expected %5s...\t\t\t\t", found?"true":"false", linked?"true":"false"); + fprintf(stderr, "ITEM %-20s: linked %5s, expected %5s...\t\t\t\t\t\t", label, + found?"true":"false", linked?"true":"false"); if(found != linked) { fprintf(stderr, "FAILED\n"); errors++; @@ -2312,27 +2812,349 @@ static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, co return errors; } -static int string_threads_join = 0; -static void *string_thread(void *arg __maybe_unused) { - int dups = 1; //(gettid() % 10); +struct thread_unittest { + int join; + DICTIONARY *dict; + int dups; +}; + +static void *unittest_dict_thread(void *arg) { + struct thread_unittest *tu = arg; for(; 1 ;) { - if(string_threads_join) + if(__atomic_load_n(&tu->join, __ATOMIC_RELAXED)) break; - STRING *s = string_strdupz("string thread checking 1234567890"); + DICT_ITEM_CONST DICTIONARY_ITEM *item = + dictionary_set_and_acquire_item_advanced(tu->dict, "dict thread checking 1234567890", + -1, NULL, 0, NULL); - for(int i = 0; i < dups ; i++) - string_dup(s); - for(int i = 0; i < dups ; i++) - string_freez(s); + dictionary_get(tu->dict, dictionary_acquired_item_name(item)); - string_freez(s); + void *t1; + dfe_start_write(tu->dict, t1) { + + // this should delete the referenced item + dictionary_del(tu->dict, t1_dfe.name); + + void *t2; + dfe_start_write(tu->dict, t2) { + // this should add another + dictionary_set(tu->dict, t2_dfe.name, NULL, 0); + + dictionary_get(tu->dict, dictionary_acquired_item_name(item)); + + // and this should delete it again + dictionary_del(tu->dict, t2_dfe.name); + } + dfe_done(t2); + + // this should fail to add it + dictionary_set(tu->dict, t1_dfe.name, NULL, 0); + dictionary_del(tu->dict, t1_dfe.name); + } + dfe_done(t1); + + for(int i = 0; i < tu->dups ; i++) { + dictionary_acquired_item_dup(tu->dict, item); + dictionary_get(tu->dict, dictionary_acquired_item_name(item)); + } + + for(int i = 0; i < tu->dups ; i++) { + dictionary_acquired_item_release(tu->dict, item); + dictionary_del(tu->dict, dictionary_acquired_item_name(item)); + } + + dictionary_acquired_item_release(tu->dict, item); + + dictionary_del(tu->dict, "dict thread checking 1234567890"); } return arg; } +static int dictionary_unittest_threads() { + + struct thread_unittest tu = { + .join = 0, + .dict = NULL, + .dups = 1, + }; + + // threads testing of dictionary + tu.dict = dictionary_create(DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); + time_t seconds_to_run = 5; + int threads_to_create = 2; + fprintf( + stderr, + "\nChecking dictionary concurrency with %d threads for %ld seconds...\n", + threads_to_create, + seconds_to_run); + + netdata_thread_t threads[threads_to_create]; + tu.join = 0; + for (int i = 0; i < threads_to_create; i++) { + char buf[100 + 1]; + snprintf(buf, 100, "dict%d", i); + netdata_thread_create( + &threads[i], + buf, + NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, + unittest_dict_thread, + &tu); + } + sleep_usec(seconds_to_run * USEC_PER_SEC); + + __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED); + for (int i = 0; i < threads_to_create; i++) { + void *retval; + netdata_thread_join(threads[i], &retval); + } + + fprintf(stderr, + "inserts %zu" + ", deletes %zu" + ", searches %zu" + ", resets %zu" + ", entries %ld" + ", referenced_items %ld" + ", pending deletions %ld" + ", check spins %zu" + ", insert spins %zu" + ", search ignores %zu" + "\n", + tu.dict->stats->ops.inserts, + tu.dict->stats->ops.deletes, + tu.dict->stats->ops.searches, + tu.dict->stats->ops.resets, + tu.dict->entries, + tu.dict->referenced_items, + tu.dict->pending_deletion_items, + tu.dict->stats->spin_locks.use, + tu.dict->stats->spin_locks.insert, + tu.dict->stats->spin_locks.search + ); + dictionary_destroy(tu.dict); + tu.dict = NULL; + + return 0; +} + +struct thread_view_unittest { + int join; + DICTIONARY *master; + DICTIONARY *view; + DICTIONARY_ITEM *item_master; + int dups; +}; + +static void *unittest_dict_master_thread(void *arg) { + struct thread_view_unittest *tv = arg; + + while(!__atomic_load_n(&tv->join, __ATOMIC_SEQ_CST)) { + if(__atomic_load_n(&tv->item_master, __ATOMIC_SEQ_CST) != NULL) + continue; + + DICTIONARY_ITEM *item = dictionary_set_and_acquire_item(tv->master, "ITEM1", "123", strlen("123") + 1); + dictionary_acquired_item_dup(tv->master, item); + dictionary_del(tv->master, "ITEM1"); + + __atomic_store_n(&tv->item_master, item, __ATOMIC_SEQ_CST); + + for(int i = 0; i < tv->dups ; i++) { + dictionary_acquired_item_dup(tv->master, item); + } + + for(int i = 0; i < tv->dups ; i++) { + dictionary_acquired_item_release(tv->master, item); + } + + dictionary_acquired_item_release(tv->master, item); + } + + return arg; +} + +static void *unittest_dict_view_thread(void *arg) { + struct thread_view_unittest *tv = arg; + + while(!__atomic_load_n(&tv->join, __ATOMIC_SEQ_CST)) { + DICTIONARY_ITEM *m_item = __atomic_load_n(&tv->item_master, __ATOMIC_SEQ_CST); + if(!m_item) continue; + + DICTIONARY_ITEM *v_item = dictionary_view_set_and_acquire_item(tv->view, "ITEM2", m_item); + dictionary_acquired_item_release(tv->master, m_item); + __atomic_store_n(&tv->item_master, NULL, __ATOMIC_SEQ_CST); + + for(int i = 0; i < tv->dups ; i++) { + dictionary_acquired_item_dup(tv->view, v_item); + } + + for(int i = 0; i < tv->dups ; i++) { + dictionary_acquired_item_release(tv->view, v_item); + } + + dictionary_del(tv->view, "ITEM2"); + + dictionary_acquired_item_release(tv->view, v_item); + } + + return arg; +} + +static int dictionary_unittest_view_threads() { + + struct thread_view_unittest tv = { + .join = 0, + .master = NULL, + .view = NULL, + .item_master = NULL, + .dups = 1, + }; + + // threads testing of dictionary + struct dictionary_stats stats = {}; + tv.master = dictionary_create_advanced(DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE, &stats); + tv.view = dictionary_create_view(tv.master); + + time_t seconds_to_run = 5; + fprintf( + stderr, + "\nChecking dictionary concurrency with 1 master and 1 view threads for %ld seconds...\n", + seconds_to_run); + + netdata_thread_t master_thread, view_thread; + tv.join = 0; + + netdata_thread_create( + &master_thread, + "master", + NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, + unittest_dict_master_thread, + &tv); + + netdata_thread_create( + &view_thread, + "view", + NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, + unittest_dict_view_thread, + &tv); + + sleep_usec(seconds_to_run * USEC_PER_SEC); + + __atomic_store_n(&tv.join, 1, __ATOMIC_RELAXED); + void *retval; + netdata_thread_join(view_thread, &retval); + netdata_thread_join(master_thread, &retval); + + fprintf(stderr, + "inserts %zu" + ", deletes %zu" + ", searches %zu" + ", resets %zu" + ", entries %ld (%ld on view)" + ", referenced_items %ld (%ld on view)" + ", pending deletions %ld (%ld on view)" + ", check spins %zu" + ", insert spins %zu" + ", search ignores %zu" + "\n", + stats.ops.inserts, + stats.ops.deletes, + stats.ops.searches, + stats.ops.resets, + tv.master->entries, tv.view->entries, + tv.master->referenced_items, tv.view->referenced_items, + tv.master->pending_deletion_items, tv.view->pending_deletion_items, + stats.spin_locks.use, + stats.spin_locks.insert, + stats.spin_locks.search + ); + dictionary_destroy(tv.master); + dictionary_destroy(tv.view); + + return 0; +} + +size_t dictionary_unittest_views(void) { + size_t errors = 0; + struct dictionary_stats stats = {}; + DICTIONARY *master = dictionary_create_advanced(DICT_OPTION_NONE, &stats); + DICTIONARY *view = dictionary_create_view(master); + + fprintf(stderr, "\n\nChecking dictionary views...\n"); + + // Add an item to both master and view, then remove the view first and the master second + fprintf(stderr, "\nPASS 1: Adding 1 item to master:\n"); + DICTIONARY_ITEM *item1_on_master = dictionary_set_and_acquire_item(master, "KEY 1", "VALUE1", strlen("VALUE1") + 1); + errors += unittest_check_dictionary("master", master, 1, 1, 0, 1, 0); + errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nPASS 1: Adding master item to view:\n"); + DICTIONARY_ITEM *item1_on_view = dictionary_view_set_and_acquire_item(view, "KEY 1 ON VIEW", item1_on_master); + errors += unittest_check_dictionary("view", view, 1, 1, 0, 1, 0); + errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nPASS 1: Deleting view item:\n"); + dictionary_del(view, "KEY 1 ON VIEW"); + errors += unittest_check_dictionary("master", master, 1, 1, 0, 1, 0); + errors += unittest_check_dictionary("view", view, 0, 0, 1, 1, 0); + errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true); + errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true); + + fprintf(stderr, "\nPASS 1: Releasing the deleted view item:\n"); + dictionary_acquired_item_release(view, item1_on_view); + errors += unittest_check_dictionary("master", master, 1, 1, 0, 1, 0); + errors += unittest_check_dictionary("view", view, 0, 0, 1, 0, 1); + errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nPASS 1: Releasing the acquired master item:\n"); + dictionary_acquired_item_release(master, item1_on_master); + errors += unittest_check_dictionary("master", master, 1, 1, 0, 0, 0); + errors += unittest_check_dictionary("view", view, 0, 0, 1, 0, 1); + errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 0, ITEM_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nPASS 1: Deleting the released master item:\n"); + dictionary_del(master, "KEY 1"); + errors += unittest_check_dictionary("master", master, 0, 0, 0, 0, 0); + errors += unittest_check_dictionary("view", view, 0, 0, 1, 0, 1); + + // The other way now: + // Add an item to both master and view, then remove the master first and verify it is deleted on the view also + fprintf(stderr, "\nPASS 2: Adding 1 item to master:\n"); + item1_on_master = dictionary_set_and_acquire_item(master, "KEY 1", "VALUE1", strlen("VALUE1") + 1); + errors += unittest_check_dictionary("master", master, 1, 1, 0, 1, 0); + errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nPASS 2: Adding master item to view:\n"); + item1_on_view = dictionary_view_set_and_acquire_item(view, "KEY 1 ON VIEW", item1_on_master); + errors += unittest_check_dictionary("view", view, 1, 1, 0, 1, 0); + errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nPASS 2: Deleting master item:\n"); + dictionary_del(master, "KEY 1"); + dictionary_version(view); + errors += unittest_check_dictionary("master", master, 0, 0, 1, 1, 0); + errors += unittest_check_dictionary("view", view, 0, 0, 1, 1, 0); + errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true); + errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true); + + fprintf(stderr, "\nPASS 2: Releasing the acquired master item:\n"); + dictionary_acquired_item_release(master, item1_on_master); + errors += unittest_check_dictionary("master", master, 0, 0, 1, 0, 1); + errors += unittest_check_dictionary("view", view, 0, 0, 1, 1, 0); + errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true); + + fprintf(stderr, "\nPASS 2: Releasing the deleted view item:\n"); + dictionary_acquired_item_release(view, item1_on_view); + errors += unittest_check_dictionary("master", master, 0, 0, 1, 0, 1); + errors += unittest_check_dictionary("view", view, 0, 0, 1, 0, 1); + + dictionary_destroy(master); + dictionary_destroy(view); + return errors; +} + int dictionary_unittest(size_t entries) { if(entries < 10) entries = 10; @@ -2344,23 +3166,28 @@ int dictionary_unittest(size_t entries) { char **values = dictionary_unittest_generate_values(entries); fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); dictionary_unittest_clone(dict, names, values, entries, &errors); fprintf(stderr, "\nCreating dictionary multi threaded, clone, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_NONE); + dict = dictionary_create(DICT_OPTION_NONE); dictionary_unittest_clone(dict, names, values, entries, &errors); fprintf(stderr, "\nCreating dictionary single threaded, non-clone, add-in-front options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT); + dict = dictionary_create( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | + DICT_OPTION_ADD_IN_FRONT); dictionary_unittest_nonclone(dict, names, values, entries, &errors); fprintf(stderr, "\nCreating dictionary multi threaded, non-clone, add-in-front options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT); + dict = dictionary_create( + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_ADD_IN_FRONT); dictionary_unittest_nonclone(dict, names, values, entries, &errors); fprintf(stderr, "\nCreating dictionary single-threaded, non-clone, don't overwrite options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + dict = dictionary_create( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | + DICT_OPTION_DONT_OVERWRITE_VALUE); dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone); dictionary_unittest_run_and_measure_time(dict, "resetting non-overwrite entries", names, values, entries, &errors, dictionary_unittest_reset_dont_overwrite_nonclone); dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, &errors, dictionary_unittest_foreach); @@ -2369,13 +3196,15 @@ int dictionary_unittest(size_t entries) { dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy); fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + dict = dictionary_create( + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone); dictionary_unittest_run_and_measure_time(dict, "walkthrough write delete this", names, values, entries, &errors, dictionary_unittest_walkthrough_delete_this); dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy); fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + dict = dictionary_create( + DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE); dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone); dictionary_unittest_run_and_measure_time(dict, "foreach write delete this", names, values, entries, &errors, dictionary_unittest_foreach_delete_this); dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop empty", names, values, 0, &errors, dictionary_unittest_foreach); @@ -2383,265 +3212,99 @@ int dictionary_unittest(size_t entries) { dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy); fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); dictionary_unittest_sorting(dict, names, values, entries, &errors); dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy); fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); dictionary_unittest_null_dfe(dict, names, values, entries, &errors); dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy); fprintf(stderr, "\nCreating dictionary single threaded, noclone, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_VALUE_LINK_DONT_CLONE); dictionary_unittest_null_dfe(dict, names, values, entries, &errors); dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy); // check reference counters { fprintf(stderr, "\nTesting reference counters:\n"); - dict = dictionary_create(DICTIONARY_FLAG_NONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE); - errors += check_dictionary(dict, 0, 0); + dict = dictionary_create(DICT_OPTION_NONE | DICT_OPTION_NAME_LINK_DONT_CLONE); + errors += unittest_check_dictionary("", dict, 0, 0, 0, 0, 0); fprintf(stderr, "\nAdding test item to dictionary and acquiring it\n"); dictionary_set(dict, "test", "ITEM1", 6); - NAME_VALUE *nv = (NAME_VALUE *)dictionary_get_and_acquire_item(dict, "test"); + DICTIONARY_ITEM *item = (DICTIONARY_ITEM *)dictionary_get_and_acquire_item(dict, "test"); - errors += check_dictionary(dict, 1, 1); - errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_NONE, true, true, true); + errors += unittest_check_dictionary("", dict, 1, 1, 0, 1, 0); + errors += unittest_check_item("ACQUIRED", dict, item, "test", "ITEM1", 1, ITEM_FLAG_NONE, true, true, true); fprintf(stderr, "\nChecking that reference counters are increased:\n"); void *t; dfe_start_read(dict, t) { - errors += check_dictionary(dict, 1, 1); - errors += - check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 2, NAME_VALUE_FLAG_NONE, true, true, true); + errors += unittest_check_dictionary("", dict, 1, 1, 0, 1, 0); + errors += unittest_check_item("ACQUIRED TRAVERSAL", dict, item, "test", "ITEM1", 2, ITEM_FLAG_NONE, true, true, true); } dfe_done(t); fprintf(stderr, "\nChecking that reference counters are decreased:\n"); - errors += check_dictionary(dict, 1, 1); - errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_NONE, true, true, true); + errors += unittest_check_dictionary("", dict, 1, 1, 0, 1, 0); + errors += unittest_check_item("ACQUIRED TRAVERSAL 2", dict, item, "test", "ITEM1", 1, ITEM_FLAG_NONE, true, true, true); fprintf(stderr, "\nDeleting the item we have acquired:\n"); dictionary_del(dict, "test"); - errors += check_dictionary(dict, 0, 1); - errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_DELETED, false, false, true); + errors += unittest_check_dictionary("", dict, 0, 0, 1, 1, 0); + errors += unittest_check_item("DELETED", dict, item, "test", "ITEM1", 1, ITEM_FLAG_DELETED, false, false, true); fprintf(stderr, "\nAdding another item with the same name of the item we deleted, while being acquired:\n"); dictionary_set(dict, "test", "ITEM2", 6); - errors += check_dictionary(dict, 1, 2); + errors += unittest_check_dictionary("", dict, 1, 1, 1, 1, 0); fprintf(stderr, "\nAcquiring the second item:\n"); - NAME_VALUE *nv2 = (NAME_VALUE *)dictionary_get_and_acquire_item(dict, "test"); - errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_DELETED, false, false, true); - errors += check_name_value_deleted_flag(dict, nv2, "test", "ITEM2", 1, NAME_VALUE_FLAG_NONE, true, true, true); + DICTIONARY_ITEM *item2 = (DICTIONARY_ITEM *)dictionary_get_and_acquire_item(dict, "test"); + errors += unittest_check_item("FIRST", dict, item, "test", "ITEM1", 1, ITEM_FLAG_DELETED, false, false, true); + errors += unittest_check_item("SECOND", dict, item2, "test", "ITEM2", 1, ITEM_FLAG_NONE, true, true, true); + errors += unittest_check_dictionary("", dict, 1, 1, 1, 2, 0); fprintf(stderr, "\nReleasing the second item (the first is still acquired):\n"); - dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)nv2); - errors += check_dictionary(dict, 1, 2); - errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_DELETED, false, false, true); - errors += check_name_value_deleted_flag(dict, nv2, "test", "ITEM2", 0, NAME_VALUE_FLAG_NONE, true, true, true); + dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)item2); + errors += unittest_check_dictionary("", dict, 1, 1, 1, 1, 0); + errors += unittest_check_item("FIRST", dict, item, "test", "ITEM1", 1, ITEM_FLAG_DELETED, false, false, true); + errors += unittest_check_item("SECOND RELEASED", dict, item2, "test", "ITEM2", 0, ITEM_FLAG_NONE, true, true, true); fprintf(stderr, "\nDeleting the second item (the first is still acquired):\n"); dictionary_del(dict, "test"); - errors += check_dictionary(dict, 0, 1); - errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_DELETED, false, false, true); + errors += unittest_check_dictionary("", dict, 0, 0, 1, 1, 0); + errors += unittest_check_item("ACQUIRED DELETED", dict, item, "test", "ITEM1", 1, ITEM_FLAG_DELETED, false, false, true); fprintf(stderr, "\nReleasing the first item (which we have already deleted):\n"); - dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)nv); - errors += check_dictionary(dict, 0, 0); + dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)item); + dfe_start_write(dict, item) ; dfe_done(item); + errors += unittest_check_dictionary("", dict, 0, 0, 1, 0, 1); fprintf(stderr, "\nAdding again the test item to dictionary and acquiring it\n"); dictionary_set(dict, "test", "ITEM1", 6); - nv = (NAME_VALUE *)dictionary_get_and_acquire_item(dict, "test"); + item = (DICTIONARY_ITEM *)dictionary_get_and_acquire_item(dict, "test"); - errors += check_dictionary(dict, 1, 1); - errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_NONE, true, true, true); + errors += unittest_check_dictionary("", dict, 1, 1, 0, 1, 0); + errors += unittest_check_item("RE-ADDITION", dict, item, "test", "ITEM1", 1, ITEM_FLAG_NONE, true, true, true); fprintf(stderr, "\nDestroying the dictionary while we have acquired an item\n"); dictionary_destroy(dict); fprintf(stderr, "Releasing the item (on a destroyed dictionary)\n"); - dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)nv); - nv = NULL; + dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)item); + item = NULL; dict = NULL; } - // check string - { - long int string_entries_starting = string_base.entries; - - fprintf(stderr, "\nChecking strings...\n"); - - STRING *s1 = string_strdupz("hello unittest"); - STRING *s2 = string_strdupz("hello unittest"); - if(s1 != s2) { - errors++; - fprintf(stderr, "ERROR: duplicating strings are not deduplicated\n"); - } - else - fprintf(stderr, "OK: duplicating string are deduplicated\n"); - - STRING *s3 = string_dup(s1); - if(s3 != s1) { - errors++; - fprintf(stderr, "ERROR: cloning strings are not deduplicated\n"); - } - else - fprintf(stderr, "OK: cloning string are deduplicated\n"); - - if(s1->refcount != 3) { - errors++; - fprintf(stderr, "ERROR: string refcount is not 3\n"); - } - else - fprintf(stderr, "OK: string refcount is 3\n"); - - STRING *s4 = string_strdupz("world unittest"); - if(s4 == s1) { - errors++; - fprintf(stderr, "ERROR: string is sharing pointers on different strings\n"); - } - else - fprintf(stderr, "OK: string is properly handling different strings\n"); - - usec_t start_ut, end_ut; - STRING **strings = mallocz(entries * sizeof(STRING *)); - - start_ut = now_realtime_usec(); - for(size_t i = 0; i < entries ;i++) { - strings[i] = string_strdupz(names[i]); - } - end_ut = now_realtime_usec(); - fprintf(stderr, "Created %zu strings in %llu usecs\n", entries, end_ut - start_ut); - - start_ut = now_realtime_usec(); - for(size_t i = 0; i < entries ;i++) { - strings[i] = string_dup(strings[i]); - } - end_ut = now_realtime_usec(); - fprintf(stderr, "Cloned %zu strings in %llu usecs\n", entries, end_ut - start_ut); - - start_ut = now_realtime_usec(); - for(size_t i = 0; i < entries ;i++) { - string_freez(strings[i]); - string_freez(strings[i]); - } - end_ut = now_realtime_usec(); - fprintf(stderr, "Freed %zu strings in %llu usecs\n", entries, end_ut - start_ut); - - freez(strings); - - if(string_base.entries != string_entries_starting + 2) { - errors++; - fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, string_base.entries); - } - else - fprintf(stderr, "OK: strings dictionary has 2 items\n"); - } - - // check 2-way merge - { - struct testcase { - char *src1; char *src2; char *expected; - } tests[] = { - { "", "", ""}, - { "a", "", "[x]"}, - { "", "a", "[x]"}, - { "a", "a", "a"}, - { "abcd", "abcd", "abcd"}, - { "foo_cs", "bar_cs", "[x]_cs"}, - { "cp_UNIQUE_INFIX_cs", "cp_unique_infix_cs", "cp_[x]_cs"}, - { "cp_UNIQUE_INFIX_ci_unique_infix_cs", "cp_unique_infix_ci_UNIQUE_INFIX_cs", "cp_[x]_cs"}, - { "foo[1234]", "foo[4321]", "foo[[x]]"}, - { NULL, NULL, NULL }, - }; - - for (struct testcase *tc = &tests[0]; tc->expected != NULL; tc++) { - STRING *src1 = string_strdupz(tc->src1); - STRING *src2 = string_strdupz(tc->src2); - STRING *expected = string_strdupz(tc->expected); - - STRING *result = string_2way_merge(src1, src2); - if (string_cmp(result, expected) != 0) { - fprintf(stderr, "string_2way_merge(\"%s\", \"%s\") -> \"%s\" (expected=\"%s\")\n", - string2str(src1), - string2str(src2), - string2str(result), - string2str(expected)); - errors++; - } - - string_freez(src1); - string_freez(src2); - string_freez(expected); - string_freez(result); - } - } - dictionary_unittest_free_char_pp(names, entries); dictionary_unittest_free_char_pp(values, entries); - { -#ifdef NETDATA_INTERNAL_CHECKS - size_t ofound_deleted_on_search = string_base.found_deleted_on_search, - ofound_available_on_search = string_base.found_available_on_search, - ofound_deleted_on_insert = string_base.found_deleted_on_insert, - ofound_available_on_insert = string_base.found_available_on_insert, - ospins = string_base.spins; -#endif - - size_t oinserts, odeletes, osearches, oentries, oreferences, omemory, oduplications, oreleases; - string_statistics(&oinserts, &odeletes, &osearches, &oentries, &oreferences, &omemory, &oduplications, &oreleases); - - time_t seconds_to_run = 5; - int threads_to_create = 2; - fprintf( - stderr, - "Checking string concurrency with %d threads for %ld seconds...\n", - threads_to_create, - seconds_to_run); - // check string concurrency - netdata_thread_t threads[threads_to_create]; - string_threads_join = 0; - for (int i = 0; i < threads_to_create; i++) { - char buf[100 + 1]; - snprintf(buf, 100, "string%d", i); - netdata_thread_create( - &threads[i], buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, string_thread, NULL); - } - sleep_usec(seconds_to_run * USEC_PER_SEC); - - string_threads_join = 1; - for (int i = 0; i < threads_to_create; i++) { - void *retval; - netdata_thread_join(threads[i], &retval); - } - - size_t inserts, deletes, searches, sentries, references, memory, duplications, releases; - string_statistics(&inserts, &deletes, &searches, &sentries, &references, &memory, &duplications, &releases); - - fprintf(stderr, "inserts %zu, deletes %zu, searches %zu, entries %zu, references %zu, memory %zu, duplications %zu, releases %zu\n", - inserts - oinserts, deletes - odeletes, searches - osearches, sentries - oentries, references - oreferences, memory - omemory, duplications - oduplications, releases - oreleases); - -#ifdef NETDATA_INTERNAL_CHECKS - size_t found_deleted_on_search = string_base.found_deleted_on_search, - found_available_on_search = string_base.found_available_on_search, - found_deleted_on_insert = string_base.found_deleted_on_insert, - found_available_on_insert = string_base.found_available_on_insert, - spins = string_base.spins; - - fprintf(stderr, "on insert: %zu ok + %zu deleted\non search: %zu ok + %zu deleted\nspins: %zu\n", - found_available_on_insert - ofound_available_on_insert, - found_deleted_on_insert - ofound_deleted_on_insert, - found_available_on_search - ofound_available_on_search, - found_deleted_on_search - ofound_deleted_on_search, - spins - ospins - ); -#endif - } + errors += dictionary_unittest_views(); + errors += dictionary_unittest_threads(); + errors += dictionary_unittest_view_threads(); fprintf(stderr, "\n%zu errors found\n", errors); return errors ? 1 : 0; diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h index 94f3999438..615d842397 100644 --- a/libnetdata/dictionary/dictionary.h +++ b/libnetdata/dictionary/dictionary.h @@ -13,19 +13,19 @@ * Names and Values in the dictionary can be cloned or linked. * In clone mode, the dictionary does all the memory management. * The default is clone for both names and values. - * Set DICTIONARY_FLAG_NAME_LINK_DONT_CLONE to link names. - * Set DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE to link names. + * Set DICT_OPTION_NAME_LINK_DONT_CLONE to link names. + * Set DICT_OPTION_VALUE_LINK_DONT_CLONE to link names. * * ORDERED * Items are ordered in the order they are added (new items are appended at the end). - * You may reverse the order by setting the flag DICTIONARY_FLAG_ADD_IN_FRONT. + * You may reverse the order by setting the flag DICT_OPTION_ADD_IN_FRONT. * * LOOKUP * The dictionary uses JudyHS to maintain a very fast randomly accessible hash table. * * MULTI-THREADED and SINGLE-THREADED * Each dictionary may be single threaded (no locks), or multi-threaded (multiple readers or one writer). - * The default is multi-threaded. Add the flag DICTIONARY_FLAG_SINGLE_THREADED for single-threaded. + * The default is multi-threaded. Add the flag DICT_OPTION_SINGLE_THREADED for single-threaded. * * WALK-THROUGH and FOREACH traversal * The dictionary can be traversed on read or write mode, either with a callback (walkthrough) or with @@ -35,110 +35,184 @@ * */ +#ifdef DICTIONARY_INTERNALS +#define DICTFE_CONST +#define DICT_ITEM_CONST +#else +#define DICTFE_CONST const +#define DICT_ITEM_CONST const +#endif + typedef struct dictionary DICTIONARY; typedef struct dictionary_item DICTIONARY_ITEM; -typedef enum dictionary_flags { - DICTIONARY_FLAG_NONE = 0, // the default is the opposite of all below - DICTIONARY_FLAG_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks) - DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy) - DICTIONARY_FLAG_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy) - DICTIONARY_FLAG_DONT_OVERWRITE_VALUE = (1 << 3), // don't overwrite values of dictionary items (default: overwrite) - DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end) +typedef enum dictionary_options { + DICT_OPTION_NONE = 0, // the default is the opposite of all below + DICT_OPTION_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks) + DICT_OPTION_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy) + DICT_OPTION_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy) + DICT_OPTION_DONT_OVERWRITE_VALUE = (1 << 3), // don't overwrite values of dictionary items (default: overwrite) + DICT_OPTION_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end) +} DICT_OPTIONS; - // to change the value of the following, you also need to change the corresponding #defines in dictionary.c - DICTIONARY_FLAG_RESERVED1 = (1 << 28), // reserved for DICTIONARY_FLAG_EXCLUSIVE_ACCESS - DICTIONARY_FLAG_RESERVED2 = (1 << 29), // reserved for DICTIONARY_FLAG_DESTROYED - DICTIONARY_FLAG_RESERVED3 = (1 << 30), // reserved for DICTIONARY_FLAG_DEFER_ALL_DELETIONS -} DICTIONARY_FLAGS; +struct dictionary_stats { + const char *name; // the name of the category + + struct { + size_t active; // the number of active dictionaries + size_t deleted; // the number of dictionaries queued for destruction + } dictionaries; + + struct { + long entries; // active items in the dictionary + long pending_deletion; // pending deletion items in the dictionary + long referenced; // referenced items in the dictionary + } items; + + struct { + size_t creations; // dictionary creations + size_t destructions; // dictionary destructions + size_t flushes; // dictionary flushes + size_t traversals; // dictionary foreach + size_t walkthroughs; // dictionary walkthrough + size_t garbage_collections; // dictionary garbage collections + size_t searches; // item searches + size_t inserts; // item inserts + size_t resets; // item resets + size_t deletes; // item deletes + } ops; + + struct { + size_t inserts; // number of times the insert callback is called + size_t conflicts; // number of times the insert conflict is called + size_t reacts; // number of times the insert react is called + size_t deletes; // number of times the insert delete is called + } callbacks; + + // memory + struct { + long indexed; // bytes of keys indexed (indication of the index size) + long values; // bytes of caller structures + long dict; // bytes of the structures dictionary needs + } memory; + + // spin locks + struct { + size_t use; // number of times a reference to item had to spin to acquire it or ignore it + size_t search; // number of times a successful search result had to be thrown away + size_t insert; // number of times an insertion to the hash table had to be repeated + } spin_locks; +}; // Create a dictionary #ifdef NETDATA_INTERNAL_CHECKS -#define dictionary_create(flags) dictionary_create_advanced_with_trace(flags, 0, __FUNCTION__, __LINE__, __FILE__); -#define dictionary_create_advanced(flags) dictionary_create_advanced_with_trace(flags, 0, __FUNCTION__, __LINE__, __FILE__); -extern DICTIONARY *dictionary_create_advanced_with_trace(DICTIONARY_FLAGS flags, size_t scratchpad_size, const char *function, size_t line, const char *file); +#define dictionary_create(options) dictionary_create_advanced_with_trace(options, NULL, __FUNCTION__, __LINE__, __FILE__) +#define dictionary_create_advanced(options, stats) dictionary_create_advanced_with_trace(options, stats, __FUNCTION__, __LINE__, __FILE__) +extern DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, const char *function, size_t line, const char *file); #else -#define dictionary_create(flags) dictionary_create_advanced(flags, 0); -extern DICTIONARY *dictionary_create_advanced(DICTIONARY_FLAGS flags, size_t scratchpad_size); +#define dictionary_create(options) dictionary_create_advanced(options, NULL); +extern DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats); #endif -extern void *dictionary_scratchpad(DICTIONARY *dict); +// Create a view on a dictionary +#ifdef NETDATA_INTERNAL_CHECKS +#define dictionary_create_view(master) dictionary_create_view_with_trace(master, __FUNCTION__, __LINE__, __FILE__) +extern DICTIONARY *dictionary_create_view_with_trace(DICTIONARY *master, const char *function, size_t line, const char *file); +#else +extern DICTIONARY *dictionary_create_view(DICTIONARY *master); +#endif // an insert callback to be called just after an item is added to the dictionary // this callback is called while the dictionary is write locked! -extern void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const char *name, void *value, void *data), void *data); +extern void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data); // a delete callback to be called just before an item is deleted forever // this callback is called while the dictionary is write locked! -extern void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_callback)(const char *name, void *value, void *data), void *data); +extern void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data); -// a merge callback to be called when DICTIONARY_FLAG_DONT_OVERWRITE_VALUE +// a merge callback to be called when DICT_OPTION_DONT_OVERWRITE_VALUE // and an item is already found in the dictionary - the dictionary does nothing else in this case // the old_value will remain in the dictionary - the new_value is ignored -extern void dictionary_register_conflict_callback(DICTIONARY *dict, void (*conflict_callback)(const char *name, void *old_value, void *new_value, void *data), void *data); +// The callback should return true if the value has been updated (it increases the dictionary version). +extern void dictionary_register_conflict_callback(DICTIONARY *dict, bool (*conflict_callback)(const DICTIONARY_ITEM *item, void *old_value, void *new_value, void *data), void *data); // a reaction callback to be called after every item insertion or conflict // after the constructors have finished and the items are fully available for use // and the dictionary is not write locked anymore -extern void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)(const char *name, void *value, void *data), void *data); +extern void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data); // Destroy a dictionary -// returns the number of bytes freed -// the returned value will not include name and value sizes if DICTIONARY_FLAG_WITH_STATISTICS is not set +// Returns the number of bytes freed +// The returned value will not include name/key sizes +// Registered delete callbacks will be run for each item in the dictionary. extern size_t dictionary_destroy(DICTIONARY *dict); +// Empties a dictionary +// Referenced items will survive, but are not offered anymore. +// Registered delete callbacks will be run for each item in the dictionary. +extern void dictionary_flush(DICTIONARY *dict); + +extern void dictionary_version_increment(DICTIONARY *dict); + +// ---------------------------------------------------------------------------- // Set an item in the dictionary +// // - if an item with the same name does not exist, create one // - if an item with the same name exists, then: -// a) if DICTIONARY_FLAG_DONT_OVERWRITE_VALUE is set, just return the existing value (ignore the new value) +// a) if DICT_OPTION_DONT_OVERWRITE_VALUE is set, just return the existing value (ignore the new value) // else b) reset the value to the new value passed at the call // -// When DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE is set, the value is linked, otherwise it is copied -// When DICTIONARY_FLAG_NAME_LINK_DONT_CLONE is set, the name is linked, otherwise it is copied +// When DICT_OPTION_VALUE_LINK_DONT_CLONE is set, the value is linked, otherwise it is copied +// When DICT_OPTION_NAME_LINK_DONT_CLONE is set, the name is linked, otherwise it is copied // -// When neither DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE nor DICTIONARY_FLAG_NAME_LINK_DONT_CLONE are set, all the +// When neither DICT_OPTION_VALUE_LINK_DONT_CLONE nor DICT_OPTION_NAME_LINK_DONT_CLONE are set, all the // memory management for names and values is done by the dictionary. // // Passing NULL as value, the dictionary will callocz() the newly allocated value, otherwise it will copy it. // Passing 0 as value_len, the dictionary will set the value to NULL (no allocations for value will be made). -extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len); +#define dictionary_set(dict, name, value, value_len) dictionary_set_advanced(dict, name, -1, value, value_len, NULL) +extern void *dictionary_set_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data); +#define dictionary_set_and_acquire_item(dict, name, value, value_len) dictionary_set_and_acquire_item_advanced(dict, name, -1, value, value_len, NULL) +extern DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_set_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data); + +// set an item in a dictionary view +#define dictionary_view_set_and_acquire_item(dict, name, master_item) dictionary_view_set_and_acquire_item_advanced(dict, name, -1, master_item) +extern DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_view_set_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, DICTIONARY_ITEM *master_item); +#define dictionary_view_set(dict, name, master_item) dictionary_view_set_advanced(dict, name, -1, master_item) +extern void *dictionary_view_set_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, DICTIONARY_ITEM *master_item); + +// ---------------------------------------------------------------------------- // Get an item from the dictionary // If it returns NULL, the item is not found -extern void *dictionary_get(DICTIONARY *dict, const char *name); +#define dictionary_get(dict, name) dictionary_get_advanced(dict, name, -1) +extern void *dictionary_get_advanced(DICTIONARY *dict, const char *name, ssize_t name_len); + +#define dictionary_get_and_acquire_item(dict, name) dictionary_get_and_acquire_item_advanced(dict, name, -1) +extern DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_get_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len); + + +// ---------------------------------------------------------------------------- // Delete an item from the dictionary -// returns 0 if the item was found and has been deleted -// returns -1 if the item was not found in the index -extern int dictionary_del(DICTIONARY *dict, const char *name); +// returns true if the item was found and has been deleted +// returns false if the item was not found in the index -extern DICTIONARY_ITEM *dictionary_get_and_acquire_item_unsafe(DICTIONARY *dict, const char *name); -extern DICTIONARY_ITEM *dictionary_get_and_acquire_item(DICTIONARY *dict, const char *name); +#define dictionary_del(dict, name) dictionary_del_advanced(dict, name, -1) +extern bool dictionary_del_advanced(DICTIONARY *dict, const char *name, ssize_t name_len); -extern DICTIONARY_ITEM *dictionary_set_and_acquire_item_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len); -extern DICTIONARY_ITEM *dictionary_set_and_acquire_item(DICTIONARY *dict, const char *name, void *value, size_t value_len); +// ---------------------------------------------------------------------------- +// reference counters management -extern void dictionary_acquired_item_release_unsafe(DICTIONARY *dict, DICTIONARY_ITEM *item); -extern void dictionary_acquired_item_release(DICTIONARY *dict, DICTIONARY_ITEM *item); +extern void dictionary_acquired_item_release(DICTIONARY *dict, DICT_ITEM_CONST DICTIONARY_ITEM *item); -extern DICTIONARY_ITEM *dictionary_acquired_item_dup(DICTIONARY_ITEM *item); -extern const char *dictionary_acquired_item_name(DICTIONARY_ITEM *item); -extern void *dictionary_acquired_item_value(DICTIONARY_ITEM *item); +extern DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_acquired_item_dup(DICTIONARY *dict, DICT_ITEM_CONST DICTIONARY_ITEM *item); -// UNSAFE functions, without locks -// to be used when the user is traversing with the right lock type -// Read lock is acquired by dictionary_walktrhough_read() and dfe_start_read() -// Write lock is acquired by dictionary_walktrhough_write() and dfe_start_write() -// For code readability, please use these macros: -#define dictionary_get_having_read_lock(dict, name) dictionary_get_unsafe(dict, name) -#define dictionary_get_having_write_lock(dict, name) dictionary_get_unsafe(dict, name) -#define dictionary_set_having_write_lock(dict, name, value, value_len) dictionary_set_unsafe(dict, name, value, value_len) -#define dictionary_del_having_write_lock(dict, name) dictionary_del_unsafe(dict, name) +extern const char *dictionary_acquired_item_name(DICT_ITEM_CONST DICTIONARY_ITEM *item); +extern void *dictionary_acquired_item_value(DICT_ITEM_CONST DICTIONARY_ITEM *item); -extern void *dictionary_get_unsafe(DICTIONARY *dict, const char *name); -extern void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len); -extern int dictionary_del_unsafe(DICTIONARY *dict, const char *name); +// ---------------------------------------------------------------------------- // Traverse (walk through) the items of the dictionary. // The order of traversal is currently the order of insertion. // @@ -153,12 +227,13 @@ extern int dictionary_del_unsafe(DICTIONARY *dict, const char *name); // #define dictionary_walkthrough_read(dict, callback, data) dictionary_walkthrough_rw(dict, 'r', callback, data) #define dictionary_walkthrough_write(dict, callback, data) dictionary_walkthrough_rw(dict, 'w', callback, data) -extern int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *value, void *data), void *data); +extern int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data); #define dictionary_sorted_walkthrough_read(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'r', callback, data) #define dictionary_sorted_walkthrough_write(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'w', callback, data) -int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data); +int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data); +// ---------------------------------------------------------------------------- // Traverse with foreach // // Use like this: @@ -173,84 +248,60 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)( // You can only delete the current item from inside a dfe_start_write() - you can add as many as you want. // -#ifdef DICTIONARY_INTERNALS -#define DICTFE_CONST -#else -#define DICTFE_CONST const -#endif - #define DICTIONARY_LOCK_READ 'r' #define DICTIONARY_LOCK_WRITE 'w' #define DICTIONARY_LOCK_REENTRANT 'z' -#define DICTIONARY_LOCK_NONE 'u' typedef DICTFE_CONST struct dictionary_foreach { + DICTIONARY *dict; // the dictionary upon we work + + DICTIONARY_ITEM *item; // the item we work on, to remember the position we are at + // this can be used with dictionary_acquired_item_dup() to + // acquire the currently working item. + DICTFE_CONST char *name; // the dictionary name of the last item used void *value; // the dictionary value of the last item used // same as the return value of dictfe_start() and dictfe_next() - // the following are for internal use only - to keep track of the point we are + size_t counter; // counts the number of iterations made, starting from zero + char rw; // the lock mode 'r' or 'w' - usec_t started_ut; // the time the caller started iterating (now_realtime_usec()) - DICTIONARY *dict; // the dictionary upon we work - void *last_item; // the item we work on, to remember the position we are at } DICTFE; #define dfe_start_read(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_READ) #define dfe_start_write(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_WRITE) #define dfe_start_reentrant(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_REENTRANT) -#define dfe_start_unsafe(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_NONE) -#define dfe_start_rw(dict, value, mode) \ - do { \ - DICTFE value ## _dfe = {}; \ - const char *value ## _name; (void)(value ## _name); (void)(value); \ - for((value) = dictionary_foreach_start_rw(&value ## _dfe, (dict), (mode)), ( value ## _name ) = value ## _dfe.name; \ - (value ## _dfe.name) ;\ - (value) = dictionary_foreach_next(&value ## _dfe), ( value ## _name ) = value ## _dfe.name) \ + +#define dfe_start_rw(dict, value, mode) \ + do { \ + DICTFE value ## _dfe = {}; \ + (void)(value); /* needed to avoid warning when looping without using this */ \ + for((value) = dictionary_foreach_start_rw(&value ## _dfe, (dict), (mode)); \ + (value ## _dfe.item) ; \ + (value) = dictionary_foreach_next(&value ## _dfe)) \ { -#define dfe_done(value) \ - } \ - dictionary_foreach_done(&value ## _dfe); \ +#define dfe_done(value) \ + } \ + dictionary_foreach_done(&value ## _dfe); \ } while(0) -extern void * dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw); -extern void * dictionary_foreach_next(DICTFE *dfe); -extern usec_t dictionary_foreach_done(DICTFE *dfe); - -// Get statistics about the dictionary -extern long int dictionary_stats_allocated_memory(DICTIONARY *dict); -extern long int dictionary_stats_entries(DICTIONARY *dict); -extern size_t dictionary_stats_version(DICTIONARY *dict); -extern size_t dictionary_stats_inserts(DICTIONARY *dict); -extern size_t dictionary_stats_searches(DICTIONARY *dict); -extern size_t dictionary_stats_deletes(DICTIONARY *dict); -extern size_t dictionary_stats_resets(DICTIONARY *dict); -extern size_t dictionary_stats_walkthroughs(DICTIONARY *dict); -extern size_t dictionary_stats_referenced_items(DICTIONARY *dict); - -extern int dictionary_unittest(size_t entries); +extern void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw); +extern void *dictionary_foreach_next(DICTFE *dfe); +extern void dictionary_foreach_done(DICTFE *dfe); // ---------------------------------------------------------------------------- -// STRING implementation +// Get statistics about the dictionary -typedef struct netdata_string STRING; -extern STRING *string_strdupz(const char *str); -extern STRING *string_dup(STRING *string); -extern void string_freez(STRING *string); -extern size_t string_strlen(STRING *string); -extern const char *string2str(STRING *string) NEVERNULL; +extern size_t dictionary_version(DICTIONARY *dict); +extern size_t dictionary_entries(DICTIONARY *dict); +extern size_t dictionary_referenced_items(DICTIONARY *dict); +extern long int dictionary_stats_for_registry(DICTIONARY *dict); -// keep common prefix/suffix and replace everything else with [x] -extern STRING *string_2way_merge(STRING *a, STRING *b); +// for all cases that the caller does not provide a stats structure, this is where they are accumulated. +extern struct dictionary_stats dictionary_stats_category_other; -static inline int string_cmp(STRING *s1, STRING *s2) { - // STRINGs are deduplicated, so the same strings have the same pointer - // when they differ, we do the typical strcmp() comparison - return (s1 == s2)?0:strcmp(string2str(s1), string2str(s2)); -} - -extern void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases); +extern int dictionary_unittest(size_t entries); // ---------------------------------------------------------------------------- // THREAD CACHE diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index b1dbad3c86..7c84359d82 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -410,6 +410,7 @@ extern char *netdata_configured_host_prefix; #include "config/appconfig.h" #include "log/log.h" #include "procfile/procfile.h" +#include "string/string.h" #include "dictionary/dictionary.h" #if defined(HAVE_LIBBPF) && !defined(__cplusplus) #include "ebpf/ebpf.h" diff --git a/libnetdata/locks/README.md b/libnetdata/locks/README.md index 9ac96a8f66..9132edc430 100644 --- a/libnetdata/locks/README.md +++ b/libnetdata/locks/README.md @@ -49,7 +49,7 @@ The library maintains a linked-list of all the lock holders (one entry per threa If any call is expected to pause the caller (ie the caller is attempting a read lock while there is a write lock in place and vice versa), the library will log something like this: ``` -RW_LOCK ON LOCK 0x0x5651c9fcce20: 4190039 'HEALTH' (function init_pending_foreach_alarms() 661@health/health.c) WANTS a 'W' lock (while holding 1 rwlocks and 1 mutexes). +RW_LOCK ON LOCK 0x0x5651c9fcce20: 4190039 'HEALTH' (function health_execute_pending_updates() 661@health/health.c) WANTS a 'W' lock (while holding 1 rwlocks and 1 mutexes). There are 7 readers and 0 writers are holding the lock: => 1: RW_LOCK: process 4190091 'WEB_SERVER[static14]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709847 usec. => 2: RW_LOCK: process 4190079 'WEB_SERVER[static6]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709869 usec. @@ -63,9 +63,9 @@ There are 7 readers and 0 writers are holding the lock: And each of the above is paired with a `GOT` log, like this: ``` -RW_LOCK ON LOCK 0x0x5651c9fcce20: 4190039 'HEALTH' (function init_pending_foreach_alarms() 661@health/health.c) GOT a 'W' lock (while holding 2 rwlocks and 1 mutexes). +RW_LOCK ON LOCK 0x0x5651c9fcce20: 4190039 'HEALTH' (function health_execute_pending_updates() 661@health/health.c) GOT a 'W' lock (while holding 2 rwlocks and 1 mutexes). There are 0 readers and 1 writers are holding the lock: - => 1: RW_LOCK: process 4190039 'HEALTH' (function init_pending_foreach_alarms() 661@health/health.c) is having 1 'W' lock for 36 usec. + => 1: RW_LOCK: process 4190039 'HEALTH' (function health_execute_pending_updates() 661@health/health.c) is having 1 'W' lock for 36 usec. ``` Keep in mind that the lock and log are not atomic. The list of callers is indicative (and sometimes just empty because the original holders of the lock, unlocked it until we had the chance to print their names). diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c index 4035031861..a209baefc9 100644 --- a/libnetdata/log/log.c +++ b/libnetdata/log/log.c @@ -850,6 +850,13 @@ void error_int( const char *prefix, const char *file __maybe_unused, const char log_unlock(); } +#ifdef NETDATA_INTERNAL_CHECKS +static void crash_netdata(void) { + // make Netdata core dump + abort(); +} +#endif + void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { // save a copy of errno - just in case this function generates a new error int __errno = errno; @@ -897,6 +904,10 @@ void fatal_int( const char *file, const char *function, const unsigned long line snprintfz(action_result, 60, "%s:%s", program_name, strncmp(thread_tag, "STREAM_RECEIVER", strlen("STREAM_RECEIVER")) ? thread_tag : "[x]"); send_statistics("FATAL", action_result, action_data); +#ifdef NETDATA_INTERNAL_CHECKS + crash_netdata(); +#endif + netdata_cleanup_and_exit(1); } diff --git a/libnetdata/required_dummies.h b/libnetdata/required_dummies.h index 3a10f1feca..3f59efbe04 100644 --- a/libnetdata/required_dummies.h +++ b/libnetdata/required_dummies.h @@ -33,6 +33,8 @@ int health_variable_lookup(STRING *variable, struct rrdcalc *rc, NETDATA_DOUBLE }; #endif +void rrdset_thread_rda_free(void){}; + // required by get_system_cpus() char *netdata_configured_host_prefix = ""; diff --git a/libnetdata/string/Makefile.am b/libnetdata/string/Makefile.am new file mode 100644 index 0000000000..161784b8f6 --- /dev/null +++ b/libnetdata/string/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/string/README.md b/libnetdata/string/README.md new file mode 100644 index 0000000000..e73ab2696c --- /dev/null +++ b/libnetdata/string/README.md @@ -0,0 +1,20 @@ + + +# STRING + +STRING provides a way to allocate and free text strings, while de-duplicating them. + +It can be used similarly to libc string functions: + + - `strdup()` and `strdupz()` become `string_strdupz()`. + - `strlen()` becomes `string_strlen()` (and it does not walkthrough the bytes of the string). + - `free()` and `freez()` become `string_freez()`. + +There is also a special `string_dup()` function that increases the reference counter of a STRING, avoiding the +index lookup to find it. + +Once there is a `STRING *`, the actual `const char *` can be accessed with `string2str()`. + +All STRING should be constant. Changing the contents of a `const char *` that has been acquired by `string2str()` should never happen. \ No newline at end of file diff --git a/libnetdata/string/string.c b/libnetdata/string/string.c new file mode 100644 index 0000000000..afa2bbe083 --- /dev/null +++ b/libnetdata/string/string.c @@ -0,0 +1,595 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" +#include + +typedef int32_t REFCOUNT; + +// ---------------------------------------------------------------------------- +// STRING implementation - dedup all STRING + +struct netdata_string { + uint32_t length; // the string length including the terminating '\0' + + REFCOUNT refcount; // how many times this string is used + // We use a signed number to be able to detect duplicate frees of a string. + // If at any point this goes below zero, we have a duplicate free. + + const char str[]; // the string itself, is appended to this structure +}; + +static struct string_hashtable { + Pvoid_t JudyHSArray; // the Judy array - hashtable + netdata_rwlock_t rwlock; // the R/W lock to protect the Judy array + + long int entries; // the number of entries in the index + long int active_references; // the number of active references alive + long int memory; // the memory used, without the JudyHS index + + size_t inserts; // the number of successful inserts to the index + size_t deletes; // the number of successful deleted from the index + size_t searches; // the number of successful searches in the index + size_t duplications; // when a string is referenced + size_t releases; // when a string is unreferenced + +#ifdef NETDATA_INTERNAL_CHECKS + // internal statistics + size_t found_deleted_on_search; + size_t found_available_on_search; + size_t found_deleted_on_insert; + size_t found_available_on_insert; + size_t spins; +#endif + +} string_base = { + .JudyHSArray = NULL, + .rwlock = NETDATA_RWLOCK_INITIALIZER, +}; + +#ifdef NETDATA_INTERNAL_CHECKS +#define string_internal_stats_add(var, val) __atomic_add_fetch(&string_base.var, val, __ATOMIC_RELAXED) +#else +#define string_internal_stats_add(var, val) do {;} while(0) +#endif + +#define string_stats_atomic_increment(var) __atomic_add_fetch(&string_base.var, 1, __ATOMIC_RELAXED) +#define string_stats_atomic_decrement(var) __atomic_sub_fetch(&string_base.var, 1, __ATOMIC_RELAXED) + +void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) { + *inserts = string_base.inserts; + *deletes = string_base.deletes; + *searches = string_base.searches; + *entries = (size_t)string_base.entries; + *references = (size_t)string_base.active_references; + *memory = (size_t)string_base.memory; + *duplications = string_base.duplications; + *releases = string_base.releases; +} + +#define string_entry_acquire(se) __atomic_add_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); +#define string_entry_release(se) __atomic_sub_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); + +static inline bool string_entry_check_and_acquire(STRING *se) { + REFCOUNT expected, desired, count = 0; + do { + count++; + + expected = __atomic_load_n(&se->refcount, __ATOMIC_SEQ_CST); + + if(expected <= 0) { + // We cannot use this. + // The reference counter reached value zero, + // so another thread is deleting this. + string_internal_stats_add(spins, count - 1); + return false; + } + + desired = expected + 1; + } + while(!__atomic_compare_exchange_n(&se->refcount, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)); + + string_internal_stats_add(spins, count - 1); + + // statistics + // string_base.active_references is altered at the in string_strdupz() and string_freez() + string_stats_atomic_increment(duplications); + + return true; +} + +STRING *string_dup(STRING *string) { + if(unlikely(!string)) return NULL; + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) <= 0)) + fatal("STRING: tried to %s() a string that is freed (it has %d references).", __FUNCTION__, string->refcount); +#endif + + string_entry_acquire(string); + + // statistics + string_stats_atomic_increment(active_references); + string_stats_atomic_increment(duplications); + + return string; +} + +// Search the index and return an ACQUIRED string entry, or NULL +static inline STRING *string_index_search(const char *str, size_t length) { + STRING *string; + + // Find the string in the index + // With a read-lock so that multiple readers can use the index concurrently. + + netdata_rwlock_rdlock(&string_base.rwlock); + + Pvoid_t *Rc; + Rc = JudyHSGet(string_base.JudyHSArray, (void *)str, length); + if(likely(Rc)) { + // found in the hash table + string = *Rc; + + if(string_entry_check_and_acquire(string)) { + // we can use this entry + string_internal_stats_add(found_available_on_search, 1); + } + else { + // this entry is about to be deleted by another thread + // do not touch it, let it go... + string = NULL; + string_internal_stats_add(found_deleted_on_search, 1); + } + } + else { + // not found in the hash table + string = NULL; + } + + string_stats_atomic_increment(searches); + netdata_rwlock_unlock(&string_base.rwlock); + + return string; +} + +// Insert a string to the index and return an ACQUIRED string entry, +// or NULL if the call needs to be retried (a deleted entry with the same key is still in the index) +// The returned entry is ACQUIRED, and it can either be: +// 1. a new item inserted, or +// 2. an item found in the index that is not currently deleted +static inline STRING *string_index_insert(const char *str, size_t length) { + STRING *string; + + netdata_rwlock_wrlock(&string_base.rwlock); + + STRING **ptr; + { + JError_t J_Error; + Pvoid_t *Rc = JudyHSIns(&string_base.JudyHSArray, (void *)str, length, &J_Error); + if (unlikely(Rc == PJERR)) { + fatal( + "STRING: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d", + str, + JU_ERRNO(&J_Error), + JU_ERRID(&J_Error)); + } + ptr = (STRING **)Rc; + } + + if (likely(*ptr == 0)) { + // a new item added to the index + size_t mem_size = sizeof(STRING) + length; + string = mallocz(mem_size); + strcpy((char *)string->str, str); + string->length = length; + string->refcount = 1; + *ptr = string; + string_base.inserts++; + string_base.entries++; + string_base.memory += (long)mem_size; + } + else { + // the item is already in the index + string = *ptr; + + if(string_entry_check_and_acquire(string)) { + // we can use this entry + string_internal_stats_add(found_available_on_insert, 1); + } + else { + // this entry is about to be deleted by another thread + // do not touch it, let it go... + string = NULL; + string_internal_stats_add(found_deleted_on_insert, 1); + } + + string_stats_atomic_increment(searches); + } + + netdata_rwlock_unlock(&string_base.rwlock); + return string; +} + +// delete an entry from the index +static inline void string_index_delete(STRING *string) { + netdata_rwlock_wrlock(&string_base.rwlock); + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) != 0)) + fatal("STRING: tried to delete a string at %s() that is already freed (it has %d references).", __FUNCTION__, string->refcount); +#endif + + bool deleted = false; + + if (likely(string_base.JudyHSArray)) { + JError_t J_Error; + int ret = JudyHSDel(&string_base.JudyHSArray, (void *)string->str, string->length, &J_Error); + if (unlikely(ret == JERR)) { + error( + "STRING: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d", + string->str, + JU_ERRNO(&J_Error), + JU_ERRID(&J_Error)); + } else + deleted = true; + } + + if (unlikely(!deleted)) + error("STRING: tried to delete '%s' that is not in the index. Ignoring it.", string->str); + else { + size_t mem_size = sizeof(STRING) + string->length; + string_base.deletes++; + string_base.entries--; + string_base.memory -= (long)mem_size; + freez(string); + } + + netdata_rwlock_unlock(&string_base.rwlock); +} + +STRING *string_strdupz(const char *str) { + if(unlikely(!str || !*str)) return NULL; + + size_t length = strlen(str) + 1; + STRING *string = string_index_search(str, length); + + while(!string) { + // The search above did not find anything, + // We loop here, because during insert we may find an entry that is being deleted by another thread. + // So, we have to let it go and retry to insert it again. + + string = string_index_insert(str, length); + } + + // statistics + string_stats_atomic_increment(active_references); + + return string; +} + +void string_freez(STRING *string) { + if(unlikely(!string)) return; + + REFCOUNT refcount = string_entry_release(string); + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(refcount < 0)) + fatal("STRING: tried to %s() a string that is already freed (it has %d references).", __FUNCTION__, string->refcount); +#endif + + if(unlikely(refcount == 0)) + string_index_delete(string); + + // statistics + string_stats_atomic_decrement(active_references); + string_stats_atomic_increment(releases); +} + +size_t string_strlen(STRING *string) { + if(unlikely(!string)) return 0; + return string->length - 1; +} + +const char *string2str(STRING *string) { + if(unlikely(!string)) return ""; + return string->str; +} + +STRING *string_2way_merge(STRING *a, STRING *b) { + static STRING *X = NULL; + + if(unlikely(!X)) { + X = string_strdupz("[x]"); + } + + if(unlikely(a == b)) return string_dup(a); + if(unlikely(a == X)) return string_dup(a); + if(unlikely(b == X)) return string_dup(b); + if(unlikely(!a)) return string_dup(X); + if(unlikely(!b)) return string_dup(X); + + size_t alen = string_strlen(a); + size_t blen = string_strlen(b); + size_t length = alen + blen + string_strlen(X) + 1; + char buf1[length + 1], buf2[length + 1], *dst1; + const char *s1, *s2; + + s1 = string2str(a); + s2 = string2str(b); + dst1 = buf1; + for( ; *s1 && *s2 && *s1 == *s2 ;s1++, s2++) + *dst1++ = *s1; + + *dst1 = '\0'; + + if(*s1 != '\0' || *s2 != '\0') { + *dst1++ = '['; + *dst1++ = 'x'; + *dst1++ = ']'; + + s1 = &(string2str(a))[alen - 1]; + s2 = &(string2str(b))[blen - 1]; + char *dst2 = &buf2[length]; + *dst2 = '\0'; + for (; *s1 && *s2 && *s1 == *s2; s1--, s2--) + *(--dst2) = *s1; + + strcpy(dst1, dst2); + } + + return string_strdupz(buf1); +} + +// ---------------------------------------------------------------------------- +// STRING unit test + +struct thread_unittest { + int join; + int dups; +}; + +static void *string_thread(void *arg) { + struct thread_unittest *tu = arg; + + for(; 1 ;) { + if(__atomic_load_n(&tu->join, __ATOMIC_RELAXED)) + break; + + STRING *s = string_strdupz("string thread checking 1234567890"); + + for(int i = 0; i < tu->dups ; i++) + string_dup(s); + + for(int i = 0; i < tu->dups ; i++) + string_freez(s); + + string_freez(s); + } + + return arg; +} + +static char **string_unittest_generate_names(size_t entries) { + char **names = mallocz(sizeof(char *) * entries); + for(size_t i = 0; i < entries ;i++) { + char buf[25 + 1] = ""; + snprintfz(buf, 25, "name.%zu.0123456789.%zu \t !@#$%%^&*(),./[]{}\\|~`", i, entries / 2 + i); + names[i] = strdupz(buf); + } + return names; +} + +static void string_unittest_free_char_pp(char **pp, size_t entries) { + for(size_t i = 0; i < entries ;i++) + freez(pp[i]); + + freez(pp); +} + +int string_unittest(size_t entries) { + size_t errors = 0; + + fprintf(stderr, "Generating %zu names and values...\n", entries); + char **names = string_unittest_generate_names(entries); + + // check string + { + long int string_entries_starting = string_base.entries; + + fprintf(stderr, "\nChecking strings...\n"); + + STRING *s1 = string_strdupz("hello unittest"); + STRING *s2 = string_strdupz("hello unittest"); + if(s1 != s2) { + errors++; + fprintf(stderr, "ERROR: duplicating strings are not deduplicated\n"); + } + else + fprintf(stderr, "OK: duplicating string are deduplicated\n"); + + STRING *s3 = string_dup(s1); + if(s3 != s1) { + errors++; + fprintf(stderr, "ERROR: cloning strings are not deduplicated\n"); + } + else + fprintf(stderr, "OK: cloning string are deduplicated\n"); + + if(s1->refcount != 3) { + errors++; + fprintf(stderr, "ERROR: string refcount is not 3\n"); + } + else + fprintf(stderr, "OK: string refcount is 3\n"); + + STRING *s4 = string_strdupz("world unittest"); + if(s4 == s1) { + errors++; + fprintf(stderr, "ERROR: string is sharing pointers on different strings\n"); + } + else + fprintf(stderr, "OK: string is properly handling different strings\n"); + + usec_t start_ut, end_ut; + STRING **strings = mallocz(entries * sizeof(STRING *)); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + strings[i] = string_strdupz(names[i]); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Created %zu strings in %llu usecs\n", entries, end_ut - start_ut); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + strings[i] = string_dup(strings[i]); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Cloned %zu strings in %llu usecs\n", entries, end_ut - start_ut); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + strings[i] = string_strdupz(string2str(strings[i])); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Found %zu existing strings in %llu usecs\n", entries, end_ut - start_ut); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + string_freez(strings[i]); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Released %zu referenced strings in %llu usecs\n", entries, end_ut - start_ut); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + string_freez(strings[i]); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Released (again) %zu referenced strings in %llu usecs\n", entries, end_ut - start_ut); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + string_freez(strings[i]); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Freed %zu strings in %llu usecs\n", entries, end_ut - start_ut); + + freez(strings); + + if(string_base.entries != string_entries_starting + 2) { + errors++; + fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, string_base.entries); + } + else + fprintf(stderr, "OK: strings dictionary has 2 items\n"); + } + + // check 2-way merge + { + struct testcase { + char *src1; char *src2; char *expected; + } tests[] = { + { "", "", ""}, + { "a", "", "[x]"}, + { "", "a", "[x]"}, + { "a", "a", "a"}, + { "abcd", "abcd", "abcd"}, + { "foo_cs", "bar_cs", "[x]_cs"}, + { "cp_UNIQUE_INFIX_cs", "cp_unique_infix_cs", "cp_[x]_cs"}, + { "cp_UNIQUE_INFIX_ci_unique_infix_cs", "cp_unique_infix_ci_UNIQUE_INFIX_cs", "cp_[x]_cs"}, + { "foo[1234]", "foo[4321]", "foo[[x]]"}, + { NULL, NULL, NULL }, + }; + + for (struct testcase *tc = &tests[0]; tc->expected != NULL; tc++) { + STRING *src1 = string_strdupz(tc->src1); + STRING *src2 = string_strdupz(tc->src2); + STRING *expected = string_strdupz(tc->expected); + + STRING *result = string_2way_merge(src1, src2); + if (string_cmp(result, expected) != 0) { + fprintf(stderr, "string_2way_merge(\"%s\", \"%s\") -> \"%s\" (expected=\"%s\")\n", + string2str(src1), + string2str(src2), + string2str(result), + string2str(expected)); + errors++; + } + + string_freez(src1); + string_freez(src2); + string_freez(expected); + string_freez(result); + } + } + + // threads testing of string + { + struct thread_unittest tu = { + .dups = 1, + .join = 0, + }; + +#ifdef NETDATA_INTERNAL_CHECKS + size_t ofound_deleted_on_search = string_base.found_deleted_on_search, + ofound_available_on_search = string_base.found_available_on_search, + ofound_deleted_on_insert = string_base.found_deleted_on_insert, + ofound_available_on_insert = string_base.found_available_on_insert, + ospins = string_base.spins; +#endif + + size_t oinserts, odeletes, osearches, oentries, oreferences, omemory, oduplications, oreleases; + string_statistics(&oinserts, &odeletes, &osearches, &oentries, &oreferences, &omemory, &oduplications, &oreleases); + + time_t seconds_to_run = 5; + int threads_to_create = 2; + fprintf( + stderr, + "Checking string concurrency with %d threads for %ld seconds...\n", + threads_to_create, + seconds_to_run); + // check string concurrency + netdata_thread_t threads[threads_to_create]; + tu.join = 0; + for (int i = 0; i < threads_to_create; i++) { + char buf[100 + 1]; + snprintf(buf, 100, "string%d", i); + netdata_thread_create( + &threads[i], buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, string_thread, &tu); + } + sleep_usec(seconds_to_run * USEC_PER_SEC); + + __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED); + for (int i = 0; i < threads_to_create; i++) { + void *retval; + netdata_thread_join(threads[i], &retval); + } + + size_t inserts, deletes, searches, sentries, references, memory, duplications, releases; + string_statistics(&inserts, &deletes, &searches, &sentries, &references, &memory, &duplications, &releases); + + fprintf(stderr, "inserts %zu, deletes %zu, searches %zu, entries %zu, references %zu, memory %zu, duplications %zu, releases %zu\n", + inserts - oinserts, deletes - odeletes, searches - osearches, sentries - oentries, references - oreferences, memory - omemory, duplications - oduplications, releases - oreleases); + +#ifdef NETDATA_INTERNAL_CHECKS + size_t found_deleted_on_search = string_base.found_deleted_on_search, + found_available_on_search = string_base.found_available_on_search, + found_deleted_on_insert = string_base.found_deleted_on_insert, + found_available_on_insert = string_base.found_available_on_insert, + spins = string_base.spins; + + fprintf(stderr, "on insert: %zu ok + %zu deleted\non search: %zu ok + %zu deleted\nspins: %zu\n", + found_available_on_insert - ofound_available_on_insert, + found_deleted_on_insert - ofound_deleted_on_insert, + found_available_on_search - ofound_available_on_search, + found_deleted_on_search - ofound_deleted_on_search, + spins - ospins + ); +#endif + } + + string_unittest_free_char_pp(names, entries); + + fprintf(stderr, "\n%zu errors found\n", errors); + return errors ? 1 : 0; +} diff --git a/libnetdata/string/string.h b/libnetdata/string/string.h new file mode 100644 index 0000000000..f527f042ff --- /dev/null +++ b/libnetdata/string/string.h @@ -0,0 +1,30 @@ + +#ifndef NETDATA_STRING_H +#define NETDATA_STRING_H 1 + +#include "../libnetdata.h" + +// ---------------------------------------------------------------------------- +// STRING implementation + +typedef struct netdata_string STRING; +extern STRING *string_strdupz(const char *str); +extern STRING *string_dup(STRING *string); +extern void string_freez(STRING *string); +extern size_t string_strlen(STRING *string); +extern const char *string2str(STRING *string) NEVERNULL; + +// keep common prefix/suffix and replace everything else with [x] +extern STRING *string_2way_merge(STRING *a, STRING *b); + +static inline int string_cmp(STRING *s1, STRING *s2) { + // STRINGs are deduplicated, so the same strings have the same pointer + // when they differ, we do the typical strcmp() comparison + return (s1 == s2)?0:strcmp(string2str(s1), string2str(s2)); +} + +extern void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases); + +extern int string_unittest(size_t entries); + +#endif diff --git a/libnetdata/threads/threads.c b/libnetdata/threads/threads.c index 58ba765a4d..df1208f204 100644 --- a/libnetdata/threads/threads.c +++ b/libnetdata/threads/threads.c @@ -29,26 +29,35 @@ const char *netdata_thread_tag(void) { // ---------------------------------------------------------------------------- // compatibility library functions +static __thread pid_t gettid_cached_tid = 0; pid_t gettid(void) { + pid_t tid = 0; + + if(likely(gettid_cached_tid > 0)) + return gettid_cached_tid; + #ifdef __FreeBSD__ - return (pid_t)pthread_getthreadid_np(); + tid = (pid_t)pthread_getthreadid_np(); #elif defined(__APPLE__) #if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060) uint64_t curthreadid; pthread_threadid_np(NULL, &curthreadid); - return (pid_t)curthreadid; + tid = (pid_t)curthreadid; #else /* __MAC_OS_X_VERSION_MIN_REQUIRED */ - return (pid_t)pthread_self; + tid = (pid_t)pthread_self; #endif /* __MAC_OS_X_VERSION_MIN_REQUIRED */ #else /* __APPLE__*/ - return (pid_t)syscall(SYS_gettid); + tid = (pid_t)syscall(SYS_gettid); #endif /* __FreeBSD__, __APPLE__*/ + + gettid_cached_tid = tid; + return tid; } // ---------------------------------------------------------------------------- @@ -97,6 +106,8 @@ void netdata_threads_init_after_fork(size_t stacksize) { // ---------------------------------------------------------------------------- // netdata_thread_create +extern void rrdset_thread_rda_free(void); + static void thread_cleanup(void *ptr) { if(netdata_thread != ptr) { NETDATA_THREAD *info = (NETDATA_THREAD *)ptr; @@ -106,6 +117,7 @@ static void thread_cleanup(void *ptr) { if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP)) info("thread with task id %d finished", gettid()); + rrdset_thread_rda_free(); thread_cache_destroy(); freez((void *)netdata_thread->tag); @@ -215,11 +227,18 @@ int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THR // ---------------------------------------------------------------------------- // netdata_thread_cancel - +#ifdef NETDATA_INTERNAL_CHECKS +int netdata_thread_cancel_with_trace(netdata_thread_t thread, int line, const char *file, const char *function) { +#else int netdata_thread_cancel(netdata_thread_t thread) { +#endif int ret = pthread_cancel(thread); if(ret != 0) +#ifdef NETDATA_INTERNAL_CHECKS + error("cannot cancel thread. pthread_cancel() failed with code %d at %d@%s, function %s()", ret, line, file, function); +#else error("cannot cancel thread. pthread_cancel() failed with code %d.", ret); +#endif return ret; } diff --git a/libnetdata/threads/threads.h b/libnetdata/threads/threads.h index e7d79d3283..854503f315 100644 --- a/libnetdata/threads/threads.h +++ b/libnetdata/threads/threads.h @@ -28,7 +28,14 @@ extern size_t netdata_threads_init(void); extern void netdata_threads_init_after_fork(size_t stacksize); extern int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg); + +#ifdef NETDATA_INTERNAL_CHECKS +#define netdata_thread_cancel(thread) netdata_thread_cancel_with_trace(thread, __LINE__, __FILE__, __FUNCTION__) +extern int netdata_thread_cancel_with_trace(netdata_thread_t thread, int line, const char *file, const char *function); +#else extern int netdata_thread_cancel(netdata_thread_t thread); +#endif + extern int netdata_thread_join(netdata_thread_t thread, void **retval); extern int netdata_thread_detach(pthread_t thread); diff --git a/ml/Dimension.h b/ml/Dimension.h index 746dcaea20..d61d385fa7 100644 --- a/ml/Dimension.h +++ b/ml/Dimension.h @@ -44,7 +44,7 @@ public: RRDDIM *getAnomalyRateRD() const { return AnomalyRateRD; } void setAnomalyRateRDName(const char *Name) const { - rrddim_set_name(AnomalyRateRD->rrdset, AnomalyRateRD, Name); + rrddim_reset_name(AnomalyRateRD->rrdset, AnomalyRateRD, Name); } virtual ~RrdDimension() {} diff --git a/registry/registry.c b/registry/registry.c index afacdfebe1..bc196b5cfa 100644 --- a/registry/registry.c +++ b/registry/registry.c @@ -108,9 +108,7 @@ static int registry_json_person_url_callback(void *entry, void *data) { } // callback for rendering MACHINE_URLs -static int registry_json_machine_url_callback(const char *name, void *entry, void *data) { - (void)name; - +static int registry_json_machine_url_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { REGISTRY_MACHINE_URL *mu = (REGISTRY_MACHINE_URL *)entry; struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data; struct web_client *w = c->w; @@ -443,8 +441,8 @@ void registry_statistics(void) { } else rrdset_next(stm); - rrddim_set(stm, "persons", registry.persons_memory + dictionary_stats_allocated_memory(registry.persons)); - rrddim_set(stm, "machines", registry.machines_memory + dictionary_stats_allocated_memory(registry.machines)); + rrddim_set(stm, "persons", registry.persons_memory + dictionary_stats_for_registry(registry.persons)); + rrddim_set(stm, "machines", registry.machines_memory + dictionary_stats_for_registry(registry.machines)); rrddim_set(stm, "urls", registry.urls_memory); rrddim_set(stm, "persons_urls", registry.persons_urls_memory); rrddim_set(stm, "machines_urls", registry.machines_urls_memory); diff --git a/registry/registry_db.c b/registry/registry_db.c index db53ff7e00..ae74aa5304 100644 --- a/registry/registry_db.c +++ b/registry/registry_db.c @@ -11,9 +11,7 @@ int registry_db_should_be_saved(void) { // ---------------------------------------------------------------------------- // INTERNAL FUNCTIONS FOR SAVING REGISTRY OBJECTS -static int registry_machine_save_url(const char *name, void *entry, void *file) { - (void)name; - +static int registry_machine_save_url(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *file) { REGISTRY_MACHINE_URL *mu = entry; FILE *fp = file; @@ -32,8 +30,7 @@ static int registry_machine_save_url(const char *name, void *entry, void *file) return ret; } -static int registry_machine_save(const char *name, void *entry, void *file) { - (void)name; +static int registry_machine_save(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *file) { REGISTRY_MACHINE *m = entry; FILE *fp = file; @@ -79,9 +76,7 @@ static inline int registry_person_save_url(void *entry, void *file) { return ret; } -static inline int registry_person_save(const char *name, void *entry, void *file) { - (void)name; - +static inline int registry_person_save(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *file) { REGISTRY_PERSON *p = entry; FILE *fp = file; diff --git a/registry/registry_init.c b/registry/registry_init.c index bae2ac5c57..ba4250ef38 100644 --- a/registry/registry_init.c +++ b/registry/registry_init.c @@ -76,8 +76,8 @@ int registry_init(void) { netdata_mutex_init(®istry.lock); // create dictionaries - registry.persons = dictionary_create(REGISTRY_DICTIONARY_FLAGS); - registry.machines = dictionary_create(REGISTRY_DICTIONARY_FLAGS); + registry.persons = dictionary_create(REGISTRY_DICTIONARY_OPTIONS); + registry.machines = dictionary_create(REGISTRY_DICTIONARY_OPTIONS); avl_init(®istry.registry_urls_root_index, registry_url_compare); // load the registry database @@ -93,9 +93,7 @@ int registry_init(void) { return 0; } -static int machine_urls_delete_callback(const char *name, void *entry, void *data) { - (void)name; - +static int machine_urls_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { REGISTRY_MACHINE *m = (REGISTRY_MACHINE *)data; (void)m; @@ -110,10 +108,7 @@ static int machine_urls_delete_callback(const char *name, void *entry, void *dat return 1; } -static int machine_delete_callback(const char *name, void *entry, void *data) { - (void)name; - (void)data; - +static int machine_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data __maybe_unused) { REGISTRY_MACHINE *m = (REGISTRY_MACHINE *)entry; int ret = dictionary_walkthrough_read(m->machine_urls, machine_urls_delete_callback, m); @@ -122,10 +117,7 @@ static int machine_delete_callback(const char *name, void *entry, void *data) { return ret + 1; } -static int registry_person_del_callback(const char *name, void *entry, void *d) { - (void)name; - (void)d; - +static int registry_person_del_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *d __maybe_unused) { REGISTRY_PERSON *p = (REGISTRY_PERSON *)entry; debug(D_REGISTRY, "Registry: registry_person_del('%s'): deleting person", p->guid); diff --git a/registry/registry_internals.h b/registry/registry_internals.h index 9e0f114775..a49b4a4c3d 100644 --- a/registry/registry_internals.h +++ b/registry/registry_internals.h @@ -8,7 +8,7 @@ #define REGISTRY_URL_FLAGS_DEFAULT 0x00 #define REGISTRY_URL_FLAGS_EXPIRED 0x01 -#define REGISTRY_DICTIONARY_FLAGS (DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED) +#define REGISTRY_DICTIONARY_OPTIONS (DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_SINGLE_THREADED) // ---------------------------------------------------------------------------- // COMMON structures diff --git a/registry/registry_machine.c b/registry/registry_machine.c index fb345aea22..414cd16d99 100644 --- a/registry/registry_machine.c +++ b/registry/registry_machine.c @@ -25,9 +25,9 @@ REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTR debug(D_REGISTRY, "registry_machine_url_allocate('%s', '%s'): indexing URL in machine", m->guid, u->url); - registry.machines_urls_memory -= dictionary_stats_allocated_memory(m->machine_urls); + registry.machines_urls_memory -= dictionary_stats_for_registry(m->machine_urls); dictionary_set(m->machine_urls, u->url, mu, sizeof(REGISTRY_MACHINE_URL)); - registry.machines_urls_memory += dictionary_stats_allocated_memory(m->machine_urls); + registry.machines_urls_memory += dictionary_stats_for_registry(m->machine_urls); registry_url_link(u); @@ -42,7 +42,7 @@ REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t whe strncpyz(m->guid, machine_guid, GUID_LEN); debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid); - m->machine_urls = dictionary_create(REGISTRY_DICTIONARY_FLAGS); + m->machine_urls = dictionary_create(REGISTRY_DICTIONARY_OPTIONS); m->first_t = m->last_t = (uint32_t)when; m->usages = 0; @@ -50,9 +50,9 @@ REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t whe registry.machines_memory += sizeof(REGISTRY_MACHINE); registry.machines_count++; - registry.machines_urls_memory -= dictionary_stats_allocated_memory(m->machine_urls); + registry.machines_urls_memory -= dictionary_stats_for_registry(m->machine_urls); dictionary_set(registry.machines, m->guid, m, sizeof(REGISTRY_MACHINE)); - registry.machines_urls_memory += dictionary_stats_allocated_memory(m->machine_urls); + registry.machines_urls_memory += dictionary_stats_for_registry(m->machine_urls); return m; } diff --git a/streaming/rrdpush.c b/streaming/rrdpush.c index 21a759dc09..514bb59c92 100644 --- a/streaming/rrdpush.c +++ b/streaming/rrdpush.c @@ -205,7 +205,7 @@ void rrdpush_send_clabels(RRDHOST *host, RRDSET *st) { // Send the current chart definition. // Assumes that collector thread has already called sender_start for mutex / buffer state. -static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { +static inline void rrdpush_send_chart_definition(RRDSET *st) { RRDHOST *host = st->rrdhost; rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_EXPOSED); @@ -260,26 +260,15 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { , rd->multiplier , rd->divisor , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":"" - , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":"" - , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":"" + , rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)?"hidden":"" + , rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":"" ); rd->exposed = 1; } + rrddim_foreach_done(rd); // send the chart local custom variables - RRDSETVAR *rs; - for(rs = st->variables; rs ;rs = rs->next) { - if(unlikely(rs->type == RRDVAR_TYPE_CALCULATED && rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR)) { - NETDATA_DOUBLE *value = (NETDATA_DOUBLE *) rs->value; - - buffer_sprintf( - host->sender->build - , "VARIABLE CHART %s = " NETDATA_DOUBLE_FORMAT "\n" - , string2str(rs->variable) - , *value - ); - } - } + rrdsetvar_print_to_streaming_custom_chart_variables(st, host->sender->build); st->upstream_resync_time = st->last_collected_time.tv_sec + (remote_clock_resync_iterations * st->update_every); } @@ -301,6 +290,7 @@ static inline bool rrdpush_send_chart_metrics_nolock(RRDSET *st, struct sender_s count_of_dimensions_written++; } } + rrddim_foreach_done(rd); buffer_strcat(host->sender->build, "END\n"); return count_of_dimensions_written != 0; @@ -315,11 +305,9 @@ bool rrdset_push_chart_definition_now(RRDSET *st) { if(unlikely(!host->rrdpush_send_enabled || !should_send_chart_matching(st))) return false; - rrdset_rdlock(st); sender_start(host->sender); - rrdpush_send_chart_definition_nolock(st); + rrdpush_send_chart_definition(st); sender_commit(host->sender); - rrdset_unlock(st); return true; } @@ -380,13 +368,13 @@ void rrdset_done_push(RRDSET *st) { host->rrdpush_sender_error_shown = 0; } - if(dictionary_stats_entries(st->rrddim_root_index) == 0) + if(dictionary_entries(st->rrddim_root_index) == 0) return; sender_start(host->sender); if(need_to_send_chart_definition(st)) - rrdpush_send_chart_definition_nolock(st); + rrdpush_send_chart_definition(st); if(rrdpush_send_chart_metrics_nolock(st, host->sender)) { // signal the sender there are more data diff --git a/streaming/rrdpush.h b/streaming/rrdpush.h index 68dab2a1a0..a6ff5ef030 100644 --- a/streaming/rrdpush.h +++ b/streaming/rrdpush.h @@ -178,7 +178,7 @@ extern void rrdpush_claimed_id(RRDHOST *host); extern int rrdpush_receiver_thread_spawn(struct web_client *w, char *url); extern void rrdpush_sender_thread_stop(RRDHOST *host); -extern void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv); +extern void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva); extern void log_stream_connection(const char *client_ip, const char *client_port, const char *api_key, const char *machine_guid, const char *host, const char *msg); extern int connect_to_one_of_destinations( struct rrdpush_destinations *destinations, diff --git a/streaming/sender.c b/streaming/sender.c index a4d7b78dfd..11d8aefc8d 100644 --- a/streaming/sender.c +++ b/streaming/sender.c @@ -124,33 +124,31 @@ static inline void rrdpush_sender_thread_close_socket(RRDHOST *host) { } } -static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *host, RRDVAR *rv) { - NETDATA_DOUBLE *value = (NETDATA_DOUBLE *)rv->value; - +static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *host, const RRDVAR_ACQUIRED *rva) { buffer_sprintf( host->sender->build , "VARIABLE HOST %s = " NETDATA_DOUBLE_FORMAT "\n" - , rrdvar_name(rv) - , *value + , rrdvar_name(rva) + , rrdvar2number(rva) ); - debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rrdvar_name(rv), *value); + debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rrdvar_name(rva), rrdvar2number(rva)); } -void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) { +void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva) { if(host->rrdpush_send_enabled && host->rrdpush_sender_spawn && __atomic_load_n(&host->rrdpush_sender_connected, __ATOMIC_SEQ_CST)) { sender_start(host->sender); - rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv); + rrdpush_sender_add_host_variable_to_buffer_nolock(host, rva); sender_commit(host->sender); } } -static int rrdpush_sender_thread_custom_host_variables_callback(const char *name __maybe_unused, void *rrdvar_ptr, void *host_ptr) { - RRDVAR *rv = (RRDVAR *)rrdvar_ptr; +static int rrdpush_sender_thread_custom_host_variables_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdvar_ptr __maybe_unused, void *host_ptr) { + const RRDVAR_ACQUIRED *rv = (const RRDVAR_ACQUIRED *)item; RRDHOST *host = (RRDHOST *)host_ptr; - if(unlikely(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR && rv->type == RRDVAR_TYPE_CALCULATED)) { + if(unlikely(rrdvar_flags(rv) & RRDVAR_FLAG_CUSTOM_HOST_VAR && rrdvar_type(rv) == RRDVAR_TYPE_CALCULATED)) { rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv); // return 1, so that the traversal will return the number of variables sent @@ -163,7 +161,7 @@ static int rrdpush_sender_thread_custom_host_variables_callback(const char *name static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) { sender_start(host->sender); - int ret = rrdvar_walkthrough_read(host->rrdvar_root_index, rrdpush_sender_thread_custom_host_variables_callback, host); + int ret = rrdvar_walkthrough_read(host->rrdvars, rrdpush_sender_thread_custom_host_variables_callback, host); (void)ret; sender_commit(host->sender); @@ -173,24 +171,18 @@ static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) { // resets all the chart, so that their definitions // will be resent to the central netdata static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) { - rrdhost_rdlock(host); - RRDSET *st; rrdset_foreach_read(st, host) { rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); st->upstream_resync_time = 0; - rrdset_rdlock(st); - RRDDIM *rd; rrddim_foreach_read(rd, st) rd->exposed = 0; - - rrdset_unlock(st); + rrddim_foreach_done(rd); } - - rrdhost_unlock(host); + rrdset_foreach_done(st); } static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) { diff --git a/tests/profile/benchmark-dictionary.c b/tests/profile/benchmark-dictionary.c index 30c098d5db..7cc9ab0ad4 100644 --- a/tests/profile/benchmark-dictionary.c +++ b/tests/profile/benchmark-dictionary.c @@ -19,8 +19,8 @@ void netdata_cleanup_and_exit(int ret) { exit(ret); } int main(int argc, char **argv) { if(argc || argv) {;} -// DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS); - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS); +// DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_WITH_STATISTICS); + DICTIONARY *dict = dictionary_create(DICT_OPTION_STATS); if(!dict) fatal("Cannot create dictionary."); struct rusage start, end; diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c index c4d9ced5f0..69f3d4367d 100644 --- a/web/api/badges/web_buffer_svg.c +++ b/web/api/badges/web_buffer_svg.c @@ -893,6 +893,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u int group = RRDR_GROUPING_AVERAGE; uint32_t options = 0x00000000; + const RRDCALC_ACQUIRED *rca = NULL; + RRDCALC *rc = NULL; + RRDSET *st = NULL; + while(url) { char *value = mystrsep(&url, "&"); if(!value || !*value) continue; @@ -957,7 +961,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u int scale = (scale_str && *scale_str)?str2i(scale_str):100; - RRDSET *st = rrdset_find(host, chart); + st = rrdset_find(host, chart); if(!st) st = rrdset_find_byname(host, chart); if(!st) { buffer_no_cacheable(w->response.data); @@ -967,9 +971,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u } st->last_accessed_time = now_realtime_sec(); - RRDCALC *rc = NULL; if(alarm) { - rc = rrdcalc_find(st, alarm); + rca = rrdcalc_from_rrdset_get(st, alarm); + rc = rrdcalc_acquired_to_rrdcalc(rca); + if (!rc) { buffer_no_cacheable(w->response.data); buffer_svg(w->response.data, "alarm not found", NAN, "", NULL, NULL, -1, scale, 0, -1, -1, NULL, NULL); @@ -1143,7 +1148,8 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u ); } - cleanup: +cleanup: + rrdcalc_from_rrdset_release(st, rca); buffer_free(dimensions); return ret; } diff --git a/web/api/exporters/shell/allmetrics_shell.c b/web/api/exporters/shell/allmetrics_shell.c index 184f44b188..0ffbac67b9 100644 --- a/web/api/exporters/shell/allmetrics_shell.c +++ b/web/api/exporters/shell/allmetrics_shell.c @@ -25,7 +25,6 @@ static inline size_t shell_name_copy(char *d, const char *s, size_t usable) { void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_string, BUFFER *wb) { analytics_log_shell(); SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT); - rrdhost_rdlock(host); // for each chart RRDSET *st; @@ -39,8 +38,6 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", rrdset_id(st), rrdset_name(st)); if(rrdset_is_available_for_viewers(st)) { - rrdset_rdlock(st); - // for each dimension RRDDIM *rd; rrddim_foreach_read(rd, st) { @@ -55,22 +52,23 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ else { if(rd->multiplier < 0 || rd->divisor < 0) n = -n; n = roundndd(n); - if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n; + if(!rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)) total += n; buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, dimension, n, rrdset_units(st)); } } } + rrddim_foreach_done(rd); total = roundndd(total); buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, total, rrdset_units(st)); - rrdset_unlock(st); } } + rrdset_foreach_done(st); buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n"); RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(!rc->rrdset) continue; char chart[SHELL_ELEMENT_MAX + 1]; @@ -90,8 +88,8 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status)); } + foreach_rrdcalc_in_rrdhost_done(rc); - rrdhost_unlock(host); simple_pattern_free(filter); } @@ -100,7 +98,6 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_string, BUFFER *wb) { analytics_log_json(); SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT); - rrdhost_rdlock(host); buffer_strcat(wb, "{"); @@ -114,8 +111,6 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s continue; if(rrdset_is_available_for_viewers(st)) { - rrdset_rdlock(st); - buffer_sprintf( wb, "%s\n" @@ -132,7 +127,7 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s rrdset_family(st), rrdset_context(st), rrdset_units(st), - (int64_t)rrdset_last_entry_t_nolock(st)); + (int64_t)rrdset_last_entry_t(st)); chart_counter++; dimension_counter = 0; @@ -161,14 +156,14 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s dimension_counter++; } } + rrddim_foreach_done(rd); buffer_strcat(wb, "\n\t\t}\n\t}"); - rrdset_unlock(st); } } + rrdset_foreach_done(st); buffer_strcat(wb, "\n}"); - rrdhost_unlock(host); simple_pattern_free(filter); } diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c index 73e4247f4e..1fc20b4935 100644 --- a/web/api/formatters/charts2json.c +++ b/web/api/formatters/charts2json.c @@ -69,7 +69,6 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived ); c = 0; - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) { if(c) buffer_strcat(wb, ","); @@ -82,13 +81,14 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived st->last_accessed_time = now; } } + rrdset_foreach_done(st); RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(rc->rrdset) alarms++; } - rrdhost_unlock(host); + foreach_rrdcalc_in_rrdhost_done(rc); buffer_sprintf(wb , "\n\t}" @@ -150,9 +150,7 @@ struct array_printer { BUFFER *wb; }; -static int print_collector_callback(const char *name, void *entry, void *data) { - (void)name; - +static int print_collector_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { struct array_printer *ap = (struct array_printer *)data; BUFFER *wb = ap->wb; struct collector *col=(struct collector *) entry; @@ -167,12 +165,11 @@ static int print_collector_callback(const char *name, void *entry, void *data) { } void chartcollectors2json(RRDHOST *host, BUFFER *wb) { - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); RRDSET *st; char name[500]; time_t now = now_realtime_sec(); - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { struct collector col = { @@ -184,7 +181,7 @@ void chartcollectors2json(RRDHOST *host, BUFFER *wb) { st->last_accessed_time = now; } } - rrdhost_unlock(host); + rrdset_foreach_done(st); struct array_printer ap = { .c = 0, .wb = wb diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index 41aee34fb8..3ebe42c99f 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -7,9 +7,7 @@ struct value_output { BUFFER *wb; }; -static int value_list_output(const char *name, void *entry, void *data) { - (void)name; - +static int value_list_output_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { struct value_output *ap = (struct value_output *)data; BUFFER *wb = ap->wb; char *output = (char *) entry; @@ -64,8 +62,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS sq[0] = '"'; } - if (should_lock) - rrdset_rdlock(r->st); buffer_sprintf(wb, "{\n" " %sapi%s: 1,\n" " %sid%s: %s%s%s,\n" @@ -83,8 +79,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS , kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_name(r->st), sq , kq, kq, r->update_every , kq, kq, r->st->update_every - , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t_nolock(r->st)) - , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t_nolock(r->st)) + , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t(r->st)) + , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t(r->st)) , kq, kq, (uint32_t)r->before , kq, kq, (uint32_t)r->after , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq @@ -94,9 +90,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq); - if (should_lock) - rrdset_unlock(r->st); - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; @@ -147,37 +140,37 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS struct value_output co = {.c = 0, .wb = wb}; - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrddim_id(rd), rrddim_name(rd)); int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrddim_id(rd), rrddim_name(rd)); dictionary_set(dict, name, output, len+1); } - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); co.c = 0; buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset)); snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset)); dictionary_set(dict, name, output, len + 1); } - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); RRDSET *st; co.c = 0; buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { st = rd->rrdset; if (st->rrdlabels) rrdlabels_walkthrough_read(st->rrdlabels, fill_formatted_callback, dict); } - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); buffer_strcat(wb, "],\n"); } diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 89efa05441..09655d281a 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -104,13 +104,11 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis (*param_list)->rd = NULL; } - RRDDIM *rd1; st->last_accessed_time = now_realtime_sec(); - rrdset_rdlock(st); - - (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t_nolock(st)); - (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st)); + (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t(st)); + (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t(st)); + RRDDIM *rd1; rrddim_foreach_read(rd1, st) { RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM)); rd->id = string_dup(rd1->id); @@ -124,8 +122,7 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis rd->next = (*param_list)->rd; (*param_list)->rd = rd; } - - rrdset_unlock(st); + rrddim_foreach_done(rd1); } void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) { diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c index d4225c98c4..7758601e29 100644 --- a/web/api/formatters/rrdset2json.c +++ b/web/api/formatters/rrdset2json.c @@ -25,10 +25,8 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation) // generate JSON for the /api/v1/chart API call void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile) { - rrdset_rdlock(st); - - time_t first_entry_t = rrdset_first_entry_t_nolock(st); - time_t last_entry_t = rrdset_last_entry_t_nolock(st); + time_t first_entry_t = rrdset_first_entry_t(st); + time_t last_entry_t = rrdset_last_entry_t(st); buffer_sprintf( wb, @@ -47,7 +45,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor "\t\t\t\"chart_type\": \"%s\",\n", rrdset_id(st), rrdset_name(st), - rrdset_type(st), + rrdset_parts_type(st), rrdset_family(st), rrdset_context(st), rrdset_title(st), @@ -90,7 +88,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor size_t dimensions = 0; RRDDIM *rd; rrddim_foreach_read(rd, st) { - if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; + if(rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; memory += sizeof(RRDDIM) + rd->memsize; @@ -105,6 +103,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor dimensions++; } + rrddim_foreach_done(rd); if(dimensions_count) *dimensions_count += dimensions; if(memory_used) *memory_used += memory; @@ -121,7 +120,8 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n"); size_t alarms = 0; RRDCALC *rc; - foreach_rrdcalc_in_rrdset(st, rc) { + netdata_rwlock_rdlock(&st->alerts.rwlock); + DOUBLE_LINKED_LIST_FOREACH_FORWARD(st->alerts.base, rc, prev, next) { buffer_sprintf( wb, "%s" @@ -136,6 +136,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor alarms++; } + netdata_rwlock_unlock(&st->alerts.rwlock); buffer_sprintf(wb, "\n\t\t\t}" ); @@ -148,6 +149,4 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_sprintf(wb, "\n\t\t}" ); - - rrdset_unlock(st); } diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 300a4429ce..44cfa5ab87 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -837,36 +837,38 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t after_wanted, time_t before_wanted, long points_wanted, RRDR_OPTIONS options, int tier) { int ret = st->update_every; - if(unlikely(!st->dimensions)) + if(unlikely(!rrdset_number_of_dimensions(st))) return ret; - rrdset_rdlock(st); - int best_tier; + RRDDIM *first_rd = NULL; + rrddim_foreach_read(first_rd, st) break; rrddim_foreach_done(first_rd); + if(!first_rd) + return ret; + int best_tier; if(options & RRDR_OPTION_SELECTED_TIER && tier >= 0 && tier < storage_tiers) best_tier = tier; - else - best_tier = rrddim_find_best_tier_for_timeframe(st->dimensions, after_wanted, before_wanted, points_wanted); + else { + best_tier = rrddim_find_best_tier_for_timeframe(first_rd, after_wanted, before_wanted, points_wanted); + } - if(!st->dimensions->tiers[best_tier]) { + if(!first_rd->tiers[best_tier]) { internal_error( true, "QUERY: tier %d on chart '%s', is not initialized", best_tier, rrdset_name(st)); } else { - ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every; + ret = (int)first_rd->tiers[best_tier]->tier_grouping * (int)st->update_every; if(unlikely(!ret)) { internal_error( true, "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d", - rrdset_name(st), st->dimensions->tiers[best_tier]->tier_grouping, st->update_every); + rrdset_name(st), first_rd->tiers[best_tier]->tier_grouping, st->update_every); ret = st->update_every; } } - rrdset_unlock(st); - return ret; } @@ -1510,7 +1512,10 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r //, size_t before_slot , const char *msg ) { - netdata_rwlock_rdlock(&r->st->rrdset_rwlock); + + time_t first_entry_t = rrdset_first_entry_t(r->st); + time_t last_entry_t = rrdset_last_entry_t(r->st); + info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), " "after (got: %zu, want: %zu, req: %ld, db: %zu), " "before (got: %zu, want: %zu, req: %ld, db: %zu), " @@ -1532,19 +1537,19 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r , (size_t)r->after , (size_t)after_wanted , after_requested - , (size_t)rrdset_first_entry_t_nolock(r->st) + , (size_t)first_entry_t // before , (size_t)r->before , (size_t)before_wanted , before_requested - , (size_t)rrdset_last_entry_t_nolock(r->st) + , (size_t)last_entry_t // duration , (size_t)(r->before - r->after + r->st->update_every) , (size_t)(before_wanted - after_wanted + r->st->update_every) , before_requested - after_requested - , (size_t)((rrdset_last_entry_t_nolock(r->st) - rrdset_first_entry_t_nolock(r->st)) + r->st->update_every) + , (size_t)((last_entry_t - first_entry_t) + r->st->update_every) // slot /* @@ -1562,7 +1567,6 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r // message , msg ); - netdata_rwlock_unlock(&r->st->rrdset_rwlock); } #endif // NETDATA_INTERNAL_CHECKS @@ -1727,10 +1731,8 @@ RRDR *rrd2rrdr( if(!context_param_list) { relative_period_requested = true; - rrdset_rdlock(st); - time_t first_entry_t = rrdset_first_entry_t_nolock(st); - time_t last_entry_t = rrdset_last_entry_t_nolock(st); - rrdset_unlock(st); + time_t first_entry_t = rrdset_first_entry_t(st); + time_t last_entry_t = rrdset_last_entry_t(st); if(first_entry_t == 0 || last_entry_t == 0) { internal_error(true, "QUERY: chart without data detected on '%s'", rrdset_name(st)); diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c index ecf4ca2ac3..09f1a12da3 100644 --- a/web/api/queries/rrdr.c +++ b/web/api/queries/rrdr.c @@ -111,7 +111,7 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_pa t = t->next; } } else - rrddim_foreach_read(rd, st) dimensions++; + dimensions = rrdset_number_of_dimensions(st); // create the rrdr RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n); @@ -121,7 +121,7 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_pa // set the hidden flag on hidden dimensions int c; for (c = 0, rd = temp_rd ? temp_rd : st->dimensions; rd; c++, rd = rd->next) { - if (unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN))) + if (unlikely(rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))) r->od[c] = RRDR_DIMENSION_HIDDEN; else r->od[c] = RRDR_DIMENSION_DEFAULT; diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index 1c80e103fa..ffc378193c 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -44,6 +44,7 @@ typedef enum rrdr_options { // internal ones - not to be exposed to the API RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate + RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION } RRDR_OPTIONS; typedef enum rrdr_value_flag { diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c index 5ff997a26b..144b072915 100644 --- a/web/api/queries/weights.c +++ b/web/api/queries/weights.c @@ -65,10 +65,7 @@ struct register_result { struct register_result *next; // used to link contexts together }; -static void register_result_insert_callback(const char *name, void *value, void *data) { - (void)name; - (void)data; - +static void register_result_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct register_result *t = (struct register_result *)value; if(t->chart_id) t->chart_id = strdupz(t->chart_id); @@ -76,9 +73,7 @@ static void register_result_insert_callback(const char *name, void *value, void if(t->dim_name) t->dim_name = strdupz(t->dim_name); } -static void register_result_delete_callback(const char *name, void *value, void *data) { - (void)name; - (void)data; +static void register_result_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct register_result *t = (struct register_result *)value; freez((void *)t->chart_id); @@ -87,7 +82,7 @@ static void register_result_delete_callback(const char *name, void *value, void } static DICTIONARY *register_result_init() { - DICTIONARY *results = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *results = dictionary_create(DICT_OPTION_SINGLE_THREADED); dictionary_register_insert_callback(results, register_result_insert_callback, results); dictionary_register_delete_callback(results, register_result_delete_callback, results); return results; @@ -261,11 +256,8 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w points, method, group, options, shifts, examined_dimensions, duration, stats); DICTIONARY *context_results = dictionary_create( - DICTIONARY_FLAG_SINGLE_THREADED - |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE - |DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - |DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - ); + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_NAME_LINK_DONT_CLONE | + DICT_OPTION_DONT_OVERWRITE_VALUE); struct register_result *t; dfe_start_read(results, t) { @@ -605,7 +597,9 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, // for each dimension RRDDIM *d; int i; - for(i = 0, d = base_rrdr->st->dimensions; d && i < base_rrdr->d; i++, d = d->next) { + rrddim_foreach_read(d, base_rrdr->st) { + if(unlikely((int)d_dfe.counter >= base_rrdr->d)) break; + i = (int)d_dfe.counter; // d_counter is provided by the dictionary // skip the not evaluated ones if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN)) @@ -650,6 +644,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, register_result(results, base_rrdr->st, d, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero); } } + rrddim_foreach_done(d); cleanup: rrdr_free(owa, high_rrdr); @@ -676,7 +671,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, usec_t started_usec = now_realtime_usec(); RRDDIM *d; - for(d = st->dimensions; d ; d = d->next) { + rrddim_foreach_read(d, st) { usec_t now_usec = now_realtime_usec(); if(now_usec - started_usec > timeout * USEC_PER_MS) return examined_dimensions; @@ -766,6 +761,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, register_result(results, st, d, pcent, flags, stats, register_zero); } + rrddim_foreach_done(d); return examined_dimensions; } @@ -787,7 +783,7 @@ static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results, usec_t started_usec = now_realtime_usec(); RRDDIM *d; - for(d = st->dimensions; d ; d = d->next) { + rrddim_foreach_read(d, st) { usec_t now_usec = now_realtime_usec(); if(now_usec - started_usec > timeout * USEC_PER_MS) return examined_dimensions; @@ -814,6 +810,7 @@ static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results, if(ret == HTTP_RESP_OK || !value_is_null || netdata_double_isnumber(average)) register_result(results, st, d, average, 0, stats, register_zero); } + rrddim_foreach_done(d); return examined_dimensions; } @@ -853,7 +850,7 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) { struct register_result *t; // count the dimensions - size_t dimensions = dictionary_stats_entries(results); + size_t dimensions = dictionary_entries(results); if(!dimensions) return 0; if(stats->max_base_high_ratio == 0.0) @@ -911,7 +908,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS WEIGHTS_STATS stats = {}; DICTIONARY *results = register_result_init(); - DICTIONARY *charts = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);; + DICTIONARY *charts = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_VALUE_LINK_DONT_CLONE);; char *error = NULL; int resp = HTTP_RESP_OK; @@ -1003,14 +1000,13 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS // dont lock here and wait for results // get the charts and run mc after RRDSET *st; - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { if(!contexts || simple_pattern_matches(contexts, rrdset_context(st))) dictionary_set(charts, rrdset_name(st), NULL, 0); } } - rrdhost_unlock(host); + rrdset_foreach_done(st); size_t examined_dimensions = 0; void *ptr; @@ -1030,11 +1026,9 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS goto cleanup; } - st = rrdset_find_byname(host, ptr_name); + st = rrdset_find_byname(host, ptr_dfe.name); // ptr_dfe.name is provided by dictionary if(!st) continue; - rrdset_rdlock(st); - switch(method) { case WEIGHTS_METHOD_ANOMALY_RATE: options |= RRDR_OPTION_ANOMALY_BIT; @@ -1066,8 +1060,6 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS &stats, register_zero); break; } - - rrdset_unlock(st); } dfe_done(ptr); diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index 8063ffc7bc..2b8e2d8e77 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -710,14 +710,13 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT); STRING *context_string = string_strdupz(context); - rrdhost_rdlock(host); rrdset_foreach_read(st1, host) { if (st1->context == context_string && (!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->rrdlabels, chart_label_key_pattern, ':')) && (!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->rrdlabels, chart_labels_filter_pattern, ':'))) build_context_param_list(owa, &context_param_list, st1); } - rrdhost_unlock(host); + rrdset_foreach_done(st1); string_freez(context_string); if (likely(context_param_list && context_param_list->rd)) // Just set the first one @@ -1054,8 +1053,7 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client * static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST *host, BUFFER *wb) { int alarm_normal = 0, alarm_warn = 0, alarm_crit = 0; RRDCALC *rc; - rrdhost_rdlock(host); - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; @@ -1070,7 +1068,7 @@ static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST alarm_normal++; } } - rrdhost_unlock(host); + foreach_rrdcalc_in_rrdhost_done(rc); buffer_sprintf(wb, "\t\t\"normal\": %d,\n", alarm_normal); buffer_sprintf(wb, "\t\t\"warning\": %d,\n", alarm_warn); buffer_sprintf(wb, "\t\t\"critical\": %d\n", alarm_crit);