opensourcemirror
/
netdata
mirror of https://github.com/netdata/netdata.git
1
0
Fork 0
Code Issues Releases Wiki Activity

netdata doubles (#13217) 

* netdata doubles

* fix cmocka test

* fix cmocka test again

* fix left-overs of long double to NETDATA_DOUBLE

* RRDDIM detached from disk representation; db settings in [db] section of netdata.conf

* update the memory before saving

* rrdset is now detached from file structures too

* on memory mode map, update the memory mapped structures on every iteration

* allow RRD_ID_LENGTH_MAX to be changed

* granularity secs, back to update every

* fix formatting

* more formatting
This commit is contained in:
Costa Tsaousis 2022-06-28 17:04:37 +03:00 committed by GitHub
parent e86cec2631
commit c3dfbe52a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
114 changed files with 1779 additions and 1495 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
aclk/aclk_rx_msgs.c
View File

@ -55,19 +55,19 @@ static int cloud_to_agent_parse(JSON_ENTRY *e)
break; break;
case JSON_NUMBER: case JSON_NUMBER:
if (!strcmp(e->name, "version")) { if (!strcmp(e->name, "version")) {
data->version = e->data.number; data->version = (int)e->data.number;
break; break;
} }
if (!strcmp(e->name, "timeout")) { if (!strcmp(e->name, "timeout")) {
data->timeout = e->data.number; data->timeout = (int)e->data.number;
break; break;
} }
if (!strcmp(e->name, "min-version")) { if (!strcmp(e->name, "min-version")) {
data->min_version = e->data.number; data->min_version = (int)e->data.number;
break; break;
} }
if (!strcmp(e->name, "max-version")) { if (!strcmp(e->name, "max-version")) {
data->max_version = e->data.number; data->max_version = (int)e->data.number;
break; break;
} }

2
cli/README.md
View File

@ -20,7 +20,7 @@ reload-health
reload-labels reload-labels
Reload all labels. Reload all labels.
save-database save-database
Save internal DB to disk for memory mode save. Save internal DB to disk for database mode save.
reopen-logs reopen-logs
Close and reopen log files. Close and reopen log files.
shutdown-agent shutdown-agent

18
collectors/apps.plugin/apps_plugin.c
View File

@ -173,7 +173,8 @@ static kernel_uint_t
global_gtime = 0; global_gtime = 0;
// the normalization ratios, as calculated by normalize_utilization() // the normalization ratios, as calculated by normalize_utilization()
double utime_fix_ratio = 1.0, NETDATA_DOUBLE
utime_fix_ratio = 1.0,
stime_fix_ratio = 1.0, stime_fix_ratio = 1.0,
gtime_fix_ratio = 1.0, gtime_fix_ratio = 1.0,
minflt_fix_ratio = 1.0, minflt_fix_ratio = 1.0,
@ -501,7 +502,8 @@ struct file_descriptor {
static int static int
all_files_len = 0, all_files_len = 0,
all_files_size = 0; all_files_size = 0;
long double currentmaxfds = 0;
long currentmaxfds = 0;
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// read users and groups from files // read users and groups from files
@ -3021,7 +3023,7 @@ static inline void aggregate_pid_fds_on_targets(struct pid_stat *p) {
reallocate_target_fds(u); reallocate_target_fds(u);
reallocate_target_fds(g); reallocate_target_fds(g);
long double currentfds = 0; long currentfds = 0;
size_t c, size = p->fds_size; size_t c, size = p->fds_size;
struct pid_fd *fds = p->fds; struct pid_fd *fds = p->fds;
for(c = 0; c < size ;c++) { for(c = 0; c < size ;c++) {
@ -3373,7 +3375,7 @@ static void normalize_utilization(struct target *root) {
gtime_fix_ratio = gtime_fix_ratio =
cutime_fix_ratio = cutime_fix_ratio =
cstime_fix_ratio = cstime_fix_ratio =
cgtime_fix_ratio = 1.0; //(double)(global_utime + global_stime) / (double)(utime + cutime + stime + cstime); cgtime_fix_ratio = 1.0; //(NETDATA_DOUBLE)(global_utime + global_stime) / (NETDATA_DOUBLE)(utime + cutime + stime + cstime);
} }
else if((global_utime + global_stime > utime + stime) && (cutime || cstime)) { else if((global_utime + global_stime > utime + stime) && (cutime || cstime)) {
// children resources are too high // children resources are too high
@ -3383,7 +3385,7 @@ static void normalize_utilization(struct target *root) {
gtime_fix_ratio = 1.0; gtime_fix_ratio = 1.0;
cutime_fix_ratio = cutime_fix_ratio =
cstime_fix_ratio = cstime_fix_ratio =
cgtime_fix_ratio = (double)((global_utime + global_stime) - (utime + stime)) / (double)(cutime + cstime); cgtime_fix_ratio = (NETDATA_DOUBLE)((global_utime + global_stime) - (utime + stime)) / (NETDATA_DOUBLE)(cutime + cstime);
} }
else if(utime || stime) { else if(utime || stime) {
// even running processes are unrealistic // even running processes are unrealistic
@ -3391,7 +3393,7 @@ static void normalize_utilization(struct target *root) {
// lower the running processes resources // lower the running processes resources
utime_fix_ratio = utime_fix_ratio =
stime_fix_ratio = stime_fix_ratio =
gtime_fix_ratio = (double)(global_utime + global_stime) / (double)(utime + stime); gtime_fix_ratio = (NETDATA_DOUBLE)(global_utime + global_stime) / (NETDATA_DOUBLE)(utime + stime);
cutime_fix_ratio = cutime_fix_ratio =
cstime_fix_ratio = cstime_fix_ratio =
cgtime_fix_ratio = 0.0; cgtime_fix_ratio = 0.0;
@ -3439,14 +3441,14 @@ static void normalize_utilization(struct target *root) {
if(utime || stime || gtime) if(utime || stime || gtime)
majflt_fix_ratio = majflt_fix_ratio =
minflt_fix_ratio = (double)(utime * utime_fix_ratio + stime * stime_fix_ratio + gtime * gtime_fix_ratio) / (double)(utime + stime + gtime); minflt_fix_ratio = (NETDATA_DOUBLE)(utime * utime_fix_ratio + stime * stime_fix_ratio + gtime * gtime_fix_ratio) / (NETDATA_DOUBLE)(utime + stime + gtime);
else else
minflt_fix_ratio = minflt_fix_ratio =
majflt_fix_ratio = 1.0; majflt_fix_ratio = 1.0;
if(cutime || cstime || cgtime) if(cutime || cstime || cgtime)
cmajflt_fix_ratio = cmajflt_fix_ratio =
cminflt_fix_ratio = (double)(cutime * cutime_fix_ratio + cstime * cstime_fix_ratio + cgtime * cgtime_fix_ratio) / (double)(cutime + cstime + cgtime); cminflt_fix_ratio = (NETDATA_DOUBLE)(cutime * cutime_fix_ratio + cstime * cstime_fix_ratio + cgtime * cgtime_fix_ratio) / (NETDATA_DOUBLE)(cutime + cstime + cgtime);
else else
cminflt_fix_ratio = cminflt_fix_ratio =
cmajflt_fix_ratio = 1.0; cmajflt_fix_ratio = 1.0;

24
collectors/cgroups.plugin/sys_fs_cgroup.c
View File

@ -835,7 +835,7 @@ struct cgroup {
unsigned long long cpu_cfs_quota; unsigned long long cpu_cfs_quota;
RRDSETVAR *chart_var_cpu_limit; RRDSETVAR *chart_var_cpu_limit;
calculated_number prev_cpu_usage; NETDATA_DOUBLE prev_cpu_usage;
char *filename_memory_limit; char *filename_memory_limit;
unsigned long long memory_limit; unsigned long long memory_limit;
@ -1012,7 +1012,7 @@ static unsigned long long calc_percentage(unsigned long long value, unsigned lon
if (total == 0) { if (total == 0) {
return 0; return 0;
} }
return (calculated_number)value / (calculated_number)total * 100; return (NETDATA_DOUBLE)value / (NETDATA_DOUBLE)total * 100;
} }
static int calc_cgroup_depth(const char *id) { static int calc_cgroup_depth(const char *id) {
@ -3686,7 +3686,7 @@ static inline int update_memory_limits(char **filename, RRDSETVAR **chart_var, u
*filename = NULL; *filename = NULL;
} }
else { else {
rrdsetvar_custom_chart_variable_set(*chart_var, (calculated_number)(*value / (1024 * 1024))); rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
return 1; return 1;
} }
} else { } else {
@ -3701,11 +3701,11 @@ static inline int update_memory_limits(char **filename, RRDSETVAR **chart_var, u
char *s = "max\n\0"; char *s = "max\n\0";
if(strcmp(s, buffer) == 0){ if(strcmp(s, buffer) == 0){
*value = UINT64_MAX; *value = UINT64_MAX;
rrdsetvar_custom_chart_variable_set(*chart_var, (calculated_number)(*value / (1024 * 1024))); rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
return 1; return 1;
} }
*value = str2ull(buffer); *value = str2ull(buffer);
rrdsetvar_custom_chart_variable_set(*chart_var, (calculated_number)(*value / (1024 * 1024))); rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
return 1; return 1;
} }
} }
@ -3814,17 +3814,17 @@ void update_cgroup_charts(int update_every) {
} }
} }
else { else {
calculated_number value = 0, quota = 0; NETDATA_DOUBLE value = 0, quota = 0;
if(likely( ((!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) && (cg->filename_cpuset_cpus || (cg->filename_cpu_cfs_period && cg->filename_cpu_cfs_quota))) if(likely( ((!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) && (cg->filename_cpuset_cpus || (cg->filename_cpu_cfs_period && cg->filename_cpu_cfs_quota)))
|| ((cg->options & CGROUP_OPTIONS_IS_UNIFIED) && cg->filename_cpu_cfs_quota))) { || ((cg->options & CGROUP_OPTIONS_IS_UNIFIED) && cg->filename_cpu_cfs_quota))) {
if(unlikely(cg->cpu_cfs_quota > 0)) if(unlikely(cg->cpu_cfs_quota > 0))
quota = (calculated_number)cg->cpu_cfs_quota / (calculated_number)cg->cpu_cfs_period; quota = (NETDATA_DOUBLE)cg->cpu_cfs_quota / (NETDATA_DOUBLE)cg->cpu_cfs_period;
if(unlikely(quota > 0 && quota < cg->cpuset_cpus)) if(unlikely(quota > 0 && quota < cg->cpuset_cpus))
value = quota * 100; value = quota * 100;
else else
value = (calculated_number)cg->cpuset_cpus * 100; value = (NETDATA_DOUBLE)cg->cpuset_cpus * 100;
} }
if(likely(value)) { if(likely(value)) {
rrdsetvar_custom_chart_variable_set(cg->chart_var_cpu_limit, value); rrdsetvar_custom_chart_variable_set(cg->chart_var_cpu_limit, value);
@ -3853,14 +3853,14 @@ void update_cgroup_charts(int update_every) {
rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE);
else else
rrddim_add(cg->st_cpu_limit, "used", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_cpu_limit, "used", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE);
cg->prev_cpu_usage = (calculated_number)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100; cg->prev_cpu_usage = (NETDATA_DOUBLE)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
} }
else else
rrdset_next(cg->st_cpu_limit); rrdset_next(cg->st_cpu_limit);
calculated_number cpu_usage = 0; NETDATA_DOUBLE cpu_usage = 0;
cpu_usage = (calculated_number)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100; cpu_usage = (NETDATA_DOUBLE)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
calculated_number cpu_used = 100 * (cpu_usage - cg->prev_cpu_usage) / (value * update_every); NETDATA_DOUBLE cpu_used = 100 * (cpu_usage - cg->prev_cpu_usage) / (value * update_every);
rrdset_isnot_obsolete(cg->st_cpu_limit); rrdset_isnot_obsolete(cg->st_cpu_limit);

2
collectors/cgroups.plugin/tests/test_doubles.c
View File

@ -109,7 +109,7 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
return NULL; return NULL;
} }
void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, calculated_number value) void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, NETDATA_DOUBLE value)
{ {
UNUSED(rs); UNUSED(rs);
UNUSED(value); UNUSED(value);

8
collectors/ebpf.plugin/ebpf_cachestat.c
View File

@ -367,23 +367,23 @@ void cachestat_update_publish(netdata_publish_cachestat_t *out, uint64_t mpa, ui
uint64_t apcl, uint64_t apd) uint64_t apcl, uint64_t apd)
{ {
// Adapted algorithm from https://github.com/iovisor/bcc/blob/master/tools/cachestat.py#L126-L138 // Adapted algorithm from https://github.com/iovisor/bcc/blob/master/tools/cachestat.py#L126-L138
calculated_number total = (calculated_number) (((long long)mpa) - ((long long)mbd)); NETDATA_DOUBLE total = (NETDATA_DOUBLE) (((long long)mpa) - ((long long)mbd));
if (total < 0) if (total < 0)
total = 0; total = 0;
calculated_number misses = (calculated_number) ( ((long long) apcl) - ((long long) apd) ); NETDATA_DOUBLE misses = (NETDATA_DOUBLE) ( ((long long) apcl) - ((long long) apd) );
if (misses < 0) if (misses < 0)
misses = 0; misses = 0;
// If hits are < 0, then its possible misses are overestimate due to possibly page cache read ahead adding // If hits are < 0, then its possible misses are overestimate due to possibly page cache read ahead adding
// more pages than needed. In this case just assume misses as total and reset hits. // more pages than needed. In this case just assume misses as total and reset hits.
calculated_number hits = total - misses; NETDATA_DOUBLE hits = total - misses;
if (hits < 0 ) { if (hits < 0 ) {
misses = total; misses = total;
hits = 0; hits = 0;
} }
calculated_number ratio = (total > 0) ? hits/total : 1; NETDATA_DOUBLE ratio = (total > 0) ? hits/total : 1;
out->ratio = (long long )(ratio*100); out->ratio = (long long )(ratio*100);
out->hit = (long long)hits; out->hit = (long long)hits;

4
collectors/ebpf.plugin/ebpf_dcstat.c
View File

@ -239,8 +239,8 @@ static inline int ebpf_dc_load_and_attach(struct dc_bpf *obj, ebpf_module_t *em)
*/ */
void dcstat_update_publish(netdata_publish_dcstat_t *out, uint64_t cache_access, uint64_t not_found) void dcstat_update_publish(netdata_publish_dcstat_t *out, uint64_t cache_access, uint64_t not_found)
{ {
calculated_number successful_access = (calculated_number) (((long long)cache_access) - ((long long)not_found)); NETDATA_DOUBLE successful_access = (NETDATA_DOUBLE) (((long long)cache_access) - ((long long)not_found));
calculated_number ratio = (cache_access) ? successful_access/(calculated_number)cache_access : 0; NETDATA_DOUBLE ratio = (cache_access) ? successful_access/(NETDATA_DOUBLE)cache_access : 0;
out->ratio = (long long )(ratio*100); out->ratio = (long long )(ratio*100);
} }

4
collectors/perf.plugin/perf_plugin.c
View File

@ -520,8 +520,8 @@ static void perf_send_metrics() {
, "instructions_per_cycle" , "instructions_per_cycle"
); );
calculated_number result = ((calculated_number)perf_events[EV_ID_INSTRUCTIONS].value / NETDATA_DOUBLE result = ((NETDATA_DOUBLE)perf_events[EV_ID_INSTRUCTIONS].value /
(calculated_number)perf_events[EV_ID_CPU_CYCLES].value) * 100.0; (NETDATA_DOUBLE)perf_events[EV_ID_CPU_CYCLES].value) * 100.0;
printf("SET %s = %lld\n" printf("SET %s = %lld\n"
, "ipc" , "ipc"
, (collected_number) result , (collected_number) result

6
collectors/plugins.d/pluginsd_parser.c
View File

@ -96,7 +96,7 @@ PARSER_RC pluginsd_disable_action(void *user)
} }
PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value) PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, NETDATA_DOUBLE value)
{ {
UNUSED(user); UNUSED(user);
@ -473,7 +473,7 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_
{ {
char *name = words[1]; char *name = words[1];
char *value = words[2]; char *value = words[2];
calculated_number v; NETDATA_DOUBLE v;
RRDSET *st = ((PARSER_USER_OBJECT *) user)->st; RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host; RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
@ -513,7 +513,7 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_
} }
char *endptr = NULL; char *endptr = NULL;
v = (calculated_number)str2ld(value, &endptr); v = (NETDATA_DOUBLE)str2ndd(value, &endptr);
if (unlikely(endptr && *endptr)) { if (unlikely(endptr && *endptr)) {
if (endptr == value) if (endptr == value)
error( error(

3
collectors/plugins.d/pluginsd_parser.h
View File

@ -30,8 +30,7 @@ extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *n
char *title, char *units, char *plugin, char *module, int priority, char *title, char *units, char *plugin, char *module, int priority,
int update_every, RRDSET_TYPE chart_type, char *options); int update_every, RRDSET_TYPE chart_type, char *options);
extern PARSER_RC pluginsd_disable_action(void *user); extern PARSER_RC pluginsd_disable_action(void *user);
extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, NETDATA_DOUBLE value);
calculated_number value);
extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type); long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type);
extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source); extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source);

2
collectors/proc.plugin/proc_diskstats.c
View File

@ -201,7 +201,7 @@ static unsigned long long int bcache_read_number_with_units(const char *filename
static int unknown_units_error = 10; static int unknown_units_error = 10;
char *end = NULL; char *end = NULL;
long double value = str2ld(buffer, &end); NETDATA_DOUBLE value = str2ndd(buffer, &end);
if(end && *end) { if(end && *end) {
if(*end == 'k') if(*end == 'k')
return (unsigned long long int)(value * 1024.0); return (unsigned long long int)(value * 1024.0);

4
collectors/proc.plugin/proc_mdstat.c
View File

@ -263,7 +263,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
word = procfile_lineword(ff, l, 3); word = procfile_lineword(ff, l, 3);
remove_trailing_chars(word, '%'); remove_trailing_chars(word, '%');
unsigned long long percentage = (unsigned long long)(str2ld(word, NULL) * 100); unsigned long long percentage = (unsigned long long)(str2ndd(word, NULL) * 100);
// possible operations: check, resync, recovery, reshape // possible operations: check, resync, recovery, reshape
// 4-th character is unique for each operation so it is checked // 4-th character is unique for each operation so it is checked
switch (procfile_lineword(ff, l, 1)[3]) { switch (procfile_lineword(ff, l, 1)[3]) {
@ -287,7 +287,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
word += 7; // skip leading "finish=" word += 7; // skip leading "finish="
if (likely(s > word)) if (likely(s > word))
raid->finish_in = (unsigned long long)(str2ld(word, NULL) * 60); raid->finish_in = (unsigned long long)(str2ndd(word, NULL) * 60);
word = procfile_lineword(ff, l, 6); word = procfile_lineword(ff, l, 6);
s = remove_trailing_chars(word, 'K'); // remove trailing "K/sec" s = remove_trailing_chars(word, 'K'); // remove trailing "K/sec"

2
collectors/proc.plugin/proc_net_dev.c
View File

@ -967,7 +967,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
d->filename_speed = NULL; d->filename_speed = NULL;
} }
else { else {
rrdsetvar_custom_chart_variable_set(d->chart_var_speed, (calculated_number) d->speed * KILOBITS_IN_A_MEGABIT); rrdsetvar_custom_chart_variable_set(d->chart_var_speed, (NETDATA_DOUBLE) d->speed * KILOBITS_IN_A_MEGABIT);
if(d->do_speed != CONFIG_BOOLEAN_NO) { if(d->do_speed != CONFIG_BOOLEAN_NO) {
if(unlikely(!d->st_speed)) { if(unlikely(!d->st_speed)) {

12
collectors/proc.plugin/proc_net_wireless.c
View File

@ -24,9 +24,9 @@ static struct netwireless {
kernel_uint_t status; kernel_uint_t status;
// Quality // Quality
calculated_number link; NETDATA_DOUBLE link;
calculated_number level; NETDATA_DOUBLE level;
calculated_number noise; NETDATA_DOUBLE noise;
// Discarded packets // Discarded packets
kernel_uint_t nwid; kernel_uint_t nwid;
@ -281,9 +281,9 @@ int do_proc_net_wireless(int update_every, usec_t dt)
} }
if (likely(do_quality != CONFIG_BOOLEAN_NO)) { if (likely(do_quality != CONFIG_BOOLEAN_NO)) {
wireless_dev->link = str2ld(procfile_lineword(ff, l, 2), NULL); wireless_dev->link = str2ndd(procfile_lineword(ff, l, 2), NULL);
wireless_dev->level = str2ld(procfile_lineword(ff, l, 3), NULL); wireless_dev->level = str2ndd(procfile_lineword(ff, l, 3), NULL);
wireless_dev->noise = str2ld(procfile_lineword(ff, l, 4), NULL); wireless_dev->noise = str2ndd(procfile_lineword(ff, l, 4), NULL);
if (unlikely(!wireless_dev->st_link)) { if (unlikely(!wireless_dev->st_link)) {
wireless_dev->st_link = rrdset_create_localhost("wireless", wireless_dev->st_link = rrdset_create_localhost("wireless",

2
collectors/proc.plugin/sys_devices_system_edac_mc.c
View File

@ -74,7 +74,7 @@ int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt) {
} }
static int do_ce = -1, do_ue = -1; static int do_ce = -1, do_ue = -1;
calculated_number ce_sum = 0, ue_sum = 0; NETDATA_DOUBLE ce_sum = 0, ue_sum = 0;
struct mc *m; struct mc *m;
if(unlikely(do_ce == -1)) { if(unlikely(do_ce == -1)) {

40
collectors/statsd.plugin/statsd.c
View File

@ -35,7 +35,7 @@
// data specific to each metric type // data specific to each metric type
typedef struct statsd_metric_gauge { typedef struct statsd_metric_gauge {
LONG_DOUBLE value; NETDATA_DOUBLE value;
} STATSD_METRIC_GAUGE; } STATSD_METRIC_GAUGE;
typedef struct statsd_metric_counter { // counter and meter typedef struct statsd_metric_counter { // counter and meter
@ -64,7 +64,7 @@ typedef struct statsd_histogram_extensions {
size_t size; size_t size;
size_t used; size_t used;
LONG_DOUBLE *values; // dynamic array of values collected NETDATA_DOUBLE *values; // dynamic array of values collected
} STATSD_METRIC_HISTOGRAM_EXTENSIONS; } STATSD_METRIC_HISTOGRAM_EXTENSIONS;
typedef struct statsd_metric_histogram { // histogram and timer typedef struct statsd_metric_histogram { // histogram and timer
@ -431,12 +431,12 @@ static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, cons
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
// statsd parsing numbers // statsd parsing numbers
static inline LONG_DOUBLE statsd_parse_float(const char *v, LONG_DOUBLE def) { static inline NETDATA_DOUBLE statsd_parse_float(const char *v, NETDATA_DOUBLE def) {
LONG_DOUBLE value; NETDATA_DOUBLE value;
if(likely(v && *v)) { if(likely(v && *v)) {
char *e = NULL; char *e = NULL;
value = str2ld(v, &e); value = str2ndd(v, &e);
if(unlikely(e && *e)) if(unlikely(e && *e))
error("STATSD: excess data '%s' after value '%s'", e, v); error("STATSD: excess data '%s' after value '%s'", e, v);
} }
@ -446,8 +446,8 @@ static inline LONG_DOUBLE statsd_parse_float(const char *v, LONG_DOUBLE def) {
return value; return value;
} }
static inline LONG_DOUBLE statsd_parse_sampling_rate(const char *v) { static inline NETDATA_DOUBLE statsd_parse_sampling_rate(const char *v) {
LONG_DOUBLE sampling_rate = statsd_parse_float(v, 1.0); NETDATA_DOUBLE sampling_rate = statsd_parse_float(v, 1.0);
if(unlikely(isless(sampling_rate, 0.001))) sampling_rate = 0.001; if(unlikely(isless(sampling_rate, 0.001))) sampling_rate = 0.001;
if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0; if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
return sampling_rate; return sampling_rate;
@ -522,7 +522,7 @@ static inline void statsd_process_counter_or_meter(STATSD_METRIC *m, const char
// magic loading of metric, without affecting anything // magic loading of metric, without affecting anything
} }
else { else {
m->counter.value += llrintl((LONG_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling)); m->counter.value += llrintndd((NETDATA_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling));
m->events++; m->events++;
m->count++; m->count++;
@ -549,18 +549,18 @@ static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const cha
// magic loading of metric, without affecting anything // magic loading of metric, without affecting anything
} }
else { else {
LONG_DOUBLE v = statsd_parse_float(value, 1.0); NETDATA_DOUBLE v = statsd_parse_float(value, 1.0);
LONG_DOUBLE sampling_rate = statsd_parse_sampling_rate(sampling); NETDATA_DOUBLE sampling_rate = statsd_parse_sampling_rate(sampling);
if(unlikely(isless(sampling_rate, 0.01))) sampling_rate = 0.01; if(unlikely(isless(sampling_rate, 0.01))) sampling_rate = 0.01;
if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0; if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
long long samples = llrintl(1.0 / sampling_rate); long long samples = llrintndd(1.0 / sampling_rate);
while(samples-- > 0) { while(samples-- > 0) {
if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) { if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) {
netdata_mutex_lock(&m->histogram.ext->mutex); netdata_mutex_lock(&m->histogram.ext->mutex);
m->histogram.ext->size += statsd.histogram_increase_step; m->histogram.ext->size += statsd.histogram_increase_step;
m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(LONG_DOUBLE) * m->histogram.ext->size); m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(NETDATA_DOUBLE) * m->histogram.ext->size);
netdata_mutex_unlock(&m->histogram.ext->mutex); netdata_mutex_unlock(&m->histogram.ext->mutex);
} }
@ -1945,21 +1945,21 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char
netdata_mutex_lock(&m->histogram.ext->mutex); netdata_mutex_lock(&m->histogram.ext->mutex);
size_t len = m->histogram.ext->used; size_t len = m->histogram.ext->used;
LONG_DOUBLE *series = m->histogram.ext->values; NETDATA_DOUBLE *series = m->histogram.ext->values;
sort_series(series, len); sort_series(series, len);
m->histogram.ext->last_min = (collected_number)roundl(series[0] * statsd.decimal_detail); m->histogram.ext->last_min = (collected_number)roundndd(series[0] * statsd.decimal_detail);
m->histogram.ext->last_max = (collected_number)roundl(series[len - 1] * statsd.decimal_detail); m->histogram.ext->last_max = (collected_number)roundndd(series[len - 1] * statsd.decimal_detail);
m->last = (collected_number)roundl(average(series, len) * statsd.decimal_detail); m->last = (collected_number)roundndd(average(series, len) * statsd.decimal_detail);
m->histogram.ext->last_median = (collected_number)roundl(median_on_sorted_series(series, len) * statsd.decimal_detail); m->histogram.ext->last_median = (collected_number)roundndd(median_on_sorted_series(series, len) * statsd.decimal_detail);
m->histogram.ext->last_stddev = (collected_number)roundl(standard_deviation(series, len) * statsd.decimal_detail); m->histogram.ext->last_stddev = (collected_number)roundndd(standard_deviation(series, len) * statsd.decimal_detail);
m->histogram.ext->last_sum = (collected_number)roundl(sum(series, len) * statsd.decimal_detail); m->histogram.ext->last_sum = (collected_number)roundndd(sum(series, len) * statsd.decimal_detail);
size_t pct_len = (size_t)floor((double)len * statsd.histogram_percentile / 100.0); size_t pct_len = (size_t)floor((double)len * statsd.histogram_percentile / 100.0);
if(pct_len < 1) if(pct_len < 1)
m->histogram.ext->last_percentile = (collected_number)(series[0] * statsd.decimal_detail); m->histogram.ext->last_percentile = (collected_number)(series[0] * statsd.decimal_detail);
else else
m->histogram.ext->last_percentile = (collected_number)roundl(series[pct_len - 1] * statsd.decimal_detail); m->histogram.ext->last_percentile = (collected_number)roundndd(series[pct_len - 1] * statsd.decimal_detail);
netdata_mutex_unlock(&m->histogram.ext->mutex); netdata_mutex_unlock(&m->histogram.ext->mutex);

78
daemon/config/README.md
View File

@ -26,20 +26,21 @@ the [web server access lists](/web/server/README.md#access-lists).
`netdata.conf` has sections stated with `[section]`. You will see the following sections: `netdata.conf` has sections stated with `[section]`. You will see the following sections:
1. `[global]` to [configure](#global-section-options) the [Netdata daemon](/daemon/README.md). 1. `[global]` to [configure](#global-section-options) the [Netdata daemon](/daemon/README.md).
2. `[directories]` to [configure](#directories-section-options) the directories used by Netdata. 2. `[db]` to [configure](#db-section-options) the database of Netdata.
3. `[logs]` to [configure](#logs-section-options) the Netdata logging. 3. `[directories]` to [configure](#directories-section-options) the directories used by Netdata.
4. `[environment variables]` to [configure](#environment-variables-section-options) the environment variables used 4. `[logs]` to [configure](#logs-section-options) the Netdata logging.
5. `[environment variables]` to [configure](#environment-variables-section-options) the environment variables used
Netdata. Netdata.
5. `[sqlite]` to [configure](#sqlite-section-options) the [Netdata daemon](/daemon/README.md) SQLite settings. 6. `[sqlite]` to [configure](#sqlite-section-options) the [Netdata daemon](/daemon/README.md) SQLite settings.
6. `[ml]` to configure settings for [machine learning](/ml/README.md). 7. `[ml]` to configure settings for [machine learning](/ml/README.md).
7. `[health]` to [configure](#health-section-options) general settings for [health monitoring](/health/README.md). 8. `[health]` to [configure](#health-section-options) general settings for [health monitoring](/health/README.md).
8. `[web]` to [configure the web server](/web/server/README.md). 9. `[web]` to [configure the web server](/web/server/README.md).
9. `[registry]` for the [Netdata registry](/registry/README.md). 10. `[registry]` for the [Netdata registry](/registry/README.md).
10. `[global statistics]` for the [Netdata registry](/registry/README.md). 11. `[global statistics]` for the [Netdata registry](/registry/README.md).
11. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md). 12. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md).
12. `[plugins]` to [configure](#plugins-section-options) which [collectors](/collectors/README.md) to use and PATH 13. `[plugins]` to [configure](#plugins-section-options) which [collectors](/collectors/README.md) to use and PATH
settings. settings.
13. `[plugin:NAME]` sections for each collector plugin, under the 14. `[plugin:NAME]` sections for each collector plugin, under the
comment [Per plugin configuration](#per-plugin-configuration). comment [Per plugin configuration](#per-plugin-configuration).
The configuration file is a `name = value` dictionary. Netdata will not complain if you set options unknown to it. When The configuration file is a `name = value` dictionary. Netdata will not complain if you set options unknown to it. When
@ -67,30 +68,35 @@ Please note that your data history will be lost if you have modified `history` p
### [global] section options ### [global] section options
| setting | default | info | | setting | default | info |
|:-------------------------------------:|:------------------------------------------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |:-------------------------------------:|:-------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| process scheduling policy | `keep` | See [Netdata process scheduling policy](/daemon/README.md#netdata-process-scheduling-policy) | | process scheduling policy | `keep` | See [Netdata process scheduling policy](/daemon/README.md#netdata-process-scheduling-policy) |
| OOM score | `0` | | | OOM score | `0` | |
| glibc malloc arena max for plugins | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). | | glibc malloc arena max for plugins | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). |
| glibc malloc arena max for Netdata | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). | | glibc malloc arena max for Netdata | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). |
| hostname | auto-detected | The hostname of the computer running Netdata. | | hostname | auto-detected | The hostname of the computer running Netdata. |
| history | `3996` | Used with `memory mode = save/map/ram/alloc`, not the default `memory mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. This setting can also be configured per chart. Check [Memory Requirements](/database/README.md) for more information. | | host access prefix | empty | This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43). |
| update every | `1` | The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md). | | timezone | auto-detected | The timezone retrieved from the environment variable |
| memory mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `page cache size` and `dbengine disk space`. <br />`save`: Netdata will save its round robin database on exit and load it on startup. <br />`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`). <br />`ram`: The round-robin database will be temporary and it will be lost when Netdata exits. <br />`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. | | run as user | `netdata` | The user Netdata will run as. |
| page cache size | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. | | pthread stack size | auto-detected | |
| dbengine disk space | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. |
| dbengine multihost disk space | 256 | Same functionality as `dbengine disk space`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. | ### [db] section options
| host access prefix | | This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43). |
| memory deduplication (ksm) | `yes` | When set to `yes`, Netdata will offer its in-memory round robin database to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm) | | setting | default | info |
| timezone | auto-detected | The timezone retrieved from the environment variable | |:----------------------------------:|:----------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| run as user | `netdata` | The user Netdata will run as. | | mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `page cache size MB` and `dbengine disk space MB`. <br />`save`: Netdata will save its round robin database on exit and load it on startup. <br />`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`). <br />`ram`: The round-robin database will be temporary and it will be lost when Netdata exits. <br />`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. |
| pthread stack size | auto-detected | | | retention | `3600` | Used with `mode = save/map/ram/alloc`, not the default `mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. Check [Memory Requirements](/database/README.md) for more information. |
| cleanup obsolete charts after seconds | `3600` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions | | update every | `1` | The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md). |
| gap when lost iterations above | `1` | | | page cache size MB | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. |
| cleanup orphan hosts after seconds | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. | | dbengine disk space MB | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. |
| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions | | dbengine multihost disk space MB | 256 | Same functionality as `dbengine disk space MB`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. |
| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. | | memory deduplication (ksm) | `yes` | When set to `yes`, Netdata will offer its in-memory round robin database and the dbengine page cache to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm) |
| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. | | cleanup obsolete charts after secs | `3600` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions |
| gap when lost iterations above | `1` | |
| cleanup orphan hosts after secs | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. |
| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions |
| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. |
| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. |
### [directories] section options ### [directories] section options

2
daemon/global_statistics.c
View File

@ -1146,7 +1146,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
if(wu->workers_cpu_registered == 0) if(wu->workers_cpu_registered == 0)
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, 0); rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, 0);
else else
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (calculated_number)wu->workers_cpu_registered )); rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (NETDATA_DOUBLE)wu->workers_cpu_registered ));
rrdset_done(wu->st_workers_cpu); rrdset_done(wu->st_workers_cpu);
} }

139
daemon/main.c
View File

@ -522,6 +522,58 @@ static void backwards_compatible_config() {
config_move(CONFIG_SECTION_STATSD, "enabled", config_move(CONFIG_SECTION_STATSD, "enabled",
CONFIG_SECTION_PLUGINS, "statsd"); CONFIG_SECTION_PLUGINS, "statsd");
config_move(CONFIG_SECTION_GLOBAL, "memory mode",
CONFIG_SECTION_DB, "mode");
config_move(CONFIG_SECTION_GLOBAL, "history",
CONFIG_SECTION_DB, "retention");
config_move(CONFIG_SECTION_GLOBAL, "update every",
CONFIG_SECTION_DB, "update every");
config_move(CONFIG_SECTION_GLOBAL, "page cache size",
CONFIG_SECTION_DB, "page cache size MB");
config_move(CONFIG_SECTION_GLOBAL, "page cache uses malloc",
CONFIG_SECTION_DB, "page cache with malloc");
config_move(CONFIG_SECTION_GLOBAL, "dbengine disk space",
CONFIG_SECTION_DB, "dbengine disk space MB");
config_move(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space",
CONFIG_SECTION_DB, "dbengine multihost disk space MB");
config_move(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)",
CONFIG_SECTION_DB, "memory deduplication (ksm)");
config_move(CONFIG_SECTION_GLOBAL, "dbengine page fetch timeout",
CONFIG_SECTION_DB, "dbengine page fetch timeout secs");
config_move(CONFIG_SECTION_GLOBAL, "dbengine page fetch retries",
CONFIG_SECTION_DB, "dbengine page fetch retries");
config_move(CONFIG_SECTION_GLOBAL, "dbengine extent pages",
CONFIG_SECTION_DB, "dbengine pages per extent");
config_move(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds",
CONFIG_SECTION_DB, "cleanup obsolete charts after secs");
config_move(CONFIG_SECTION_GLOBAL, "gap when lost iterations above",
CONFIG_SECTION_DB, "gap when lost iterations above");
config_move(CONFIG_SECTION_GLOBAL, "cleanup orphan hosts after seconds",
CONFIG_SECTION_DB, "cleanup orphan hosts after secs");
config_move(CONFIG_SECTION_GLOBAL, "delete obsolete charts files",
CONFIG_SECTION_DB, "delete obsolete charts files");
config_move(CONFIG_SECTION_GLOBAL, "delete orphan hosts files",
CONFIG_SECTION_DB, "delete orphan hosts files");
config_move(CONFIG_SECTION_GLOBAL, "enable zero metrics",
CONFIG_SECTION_DB, "enable zero metrics");
} }
static void get_netdata_configured_variables() { static void get_netdata_configured_variables() {
@ -539,28 +591,40 @@ static void get_netdata_configured_variables() {
debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname); debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname);
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// get default database size // get default database update frequency
default_rrd_history_entries = (int) config_get_number(CONFIG_SECTION_GLOBAL, "history", align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", UPDATE_EVERY);
if(default_rrd_update_every < 1 || default_rrd_update_every > 600) {
long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries); error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY);
if(h != default_rrd_history_entries) { default_rrd_update_every = UPDATE_EVERY;
config_set_number(CONFIG_SECTION_GLOBAL, "history", h); config_set_number(CONFIG_SECTION_DB, "update every", default_rrd_update_every);
default_rrd_history_entries = (int)h;
}
if(default_rrd_history_entries < 5 || default_rrd_history_entries > RRD_HISTORY_ENTRIES_MAX) {
error("Invalid history entries %d given. Defaulting to %d.", default_rrd_history_entries, RRD_DEFAULT_HISTORY_ENTRIES);
default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES;
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// get default database update frequency // get default memory mode for the database
default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_GLOBAL, "update every", UPDATE_EVERY); {
if(default_rrd_update_every < 1 || default_rrd_update_every > 600) { const char *mode = config_get(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY_MAX); default_rrd_memory_mode = rrd_memory_mode_id(mode);
default_rrd_update_every = UPDATE_EVERY; if(strcmp(mode, rrd_memory_mode_name(default_rrd_memory_mode)) != 0) {
error("Invalid memory mode '%s' given. Using '%s'", mode, rrd_memory_mode_name(default_rrd_memory_mode));
config_set(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
}
}
// ------------------------------------------------------------------------
// get default database size
if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) {
default_rrd_history_entries = (int)config_get_number(
CONFIG_SECTION_DB, "retention",
align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES));
long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries);
if (h != default_rrd_history_entries) {
config_set_number(CONFIG_SECTION_DB, "retention", h);
default_rrd_history_entries = (int)h;
}
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
@ -582,39 +646,38 @@ static void get_netdata_configured_variables() {
netdata_configured_primary_plugins_dir = plugin_directories[PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH]; netdata_configured_primary_plugins_dir = plugin_directories[PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH];
} }
// ------------------------------------------------------------------------
// get default memory mode for the database
default_rrd_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_GLOBAL, "memory mode", rrd_memory_mode_name(default_rrd_memory_mode)));
#ifdef ENABLE_DBENGINE #ifdef ENABLE_DBENGINE
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// get default Database Engine page cache size in MiB // get default Database Engine page cache size in MiB
db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_GLOBAL, "page cache uses malloc", CONFIG_BOOLEAN_NO); db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "page cache with malloc", CONFIG_BOOLEAN_NO);
default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "page cache size", default_rrdeng_page_cache_mb); default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "page cache size MB", default_rrdeng_page_cache_mb);
if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) { if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB); error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB; default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
config_set_number(CONFIG_SECTION_DB, "page cache size MB", default_rrdeng_page_cache_mb);
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// get default Database Engine disk space quota in MiB // get default Database Engine disk space quota in MiB
default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine disk space", default_rrdeng_disk_quota_mb); default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB); error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB; default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
} }
default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", compute_multidb_diskspace()); default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());
if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb); error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);
default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb; default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb);
} }
#else #else
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use memory mode ram instead."); error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead.");
default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; default_rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
} }
#endif #endif
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
@ -626,11 +689,12 @@ static void get_netdata_configured_variables() {
// get KSM settings // get KSM settings
#ifdef MADV_MERGEABLE #ifdef MADV_MERGEABLE
enable_ksm = config_get_boolean(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", enable_ksm); enable_ksm = config_get_boolean(CONFIG_SECTION_DB, "memory deduplication (ksm)", enable_ksm);
#endif #endif
// -------------------------------------------------------------------- // --------------------------------------------------------------------
// metric correlations // metric correlations
enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations); enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations);
default_metric_correlations_method = mc_string_to_method(config_get(CONFIG_SECTION_GLOBAL, "metric correlations method", mc_method_to_string(default_metric_correlations_method))); default_metric_correlations_method = mc_string_to_method(config_get(CONFIG_SECTION_GLOBAL, "metric correlations method", mc_method_to_string(default_metric_correlations_method)));
@ -1232,39 +1296,47 @@ int main(int argc, char **argv) {
// -------------------------------------------------------------------- // --------------------------------------------------------------------
// get log filenames and settings // get log filenames and settings
log_init(); log_init();
error_log_limit_unlimited(); error_log_limit_unlimited();
// initialize the log files // initialize the log files
open_all_log_files(); open_all_log_files();
#ifdef ENABLE_DBENGINE #ifdef ENABLE_DBENGINE
default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine page fetch timeout", PAGE_CACHE_FETCH_WAIT_TIMEOUT); default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT);
if (default_rrdeng_page_fetch_timeout < 1) { if (default_rrdeng_page_fetch_timeout < 1) {
info("\"dbengine page fetch timeout\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_timeout); info("'dbengine page fetch timeout secs' cannot be %d, using 1", default_rrdeng_page_fetch_timeout);
default_rrdeng_page_fetch_timeout = 1; default_rrdeng_page_fetch_timeout = 1;
config_set_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", default_rrdeng_page_fetch_timeout);
} }
default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES); default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES);
if (default_rrdeng_page_fetch_retries < 1) { if (default_rrdeng_page_fetch_retries < 1) {
info("\"dbengine page fetch retries\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_retries); info("\"dbengine page fetch retries\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_retries);
default_rrdeng_page_fetch_retries = 1; default_rrdeng_page_fetch_retries = 1;
config_set_number(CONFIG_SECTION_DB, "dbengine page fetch retries", default_rrdeng_page_fetch_retries);
} }
#endif #endif
get_system_timezone(); get_system_timezone();
// -------------------------------------------------------------------- // --------------------------------------------------------------------
// get the certificate and start security // get the certificate and start security
#ifdef ENABLE_HTTPS #ifdef ENABLE_HTTPS
security_init(); security_init();
#endif #endif
// -------------------------------------------------------------------- // --------------------------------------------------------------------
// This is the safest place to start the SILENCERS structure // This is the safest place to start the SILENCERS structure
set_silencers_filename(); set_silencers_filename();
health_initialize_global_silencers(); health_initialize_global_silencers();
// -------------------------------------------------------------------- // --------------------------------------------------------------------
// Initialize ML configuration // Initialize ML configuration
ml_init(); ml_init();
// -------------------------------------------------------------------- // --------------------------------------------------------------------
@ -1272,9 +1344,11 @@ int main(int argc, char **argv) {
// block signals while initializing threads. // block signals while initializing threads.
// this causes the threads to block signals. // this causes the threads to block signals.
signals_block(); signals_block();
// setup the signals we want to use // setup the signals we want to use
signals_init(); signals_init();
// setup threads configs // setup threads configs
@ -1303,6 +1377,7 @@ int main(int argc, char **argv) {
if(web_server_mode != WEB_SERVER_MODE_NONE) if(web_server_mode != WEB_SERVER_MODE_NONE)
api_listen_sockets_setup(); api_listen_sockets_setup();
} }
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
@ -1386,7 +1461,7 @@ int main(int argc, char **argv) {
web_server_config_options(); web_server_config_options();
netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_GLOBAL, "enable zero metrics", CONFIG_BOOLEAN_NO); netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_DB, "enable zero metrics", CONFIG_BOOLEAN_NO);
set_late_global_environment(); set_late_global_environment();

251
daemon/unit_test.c
View File

@ -4,7 +4,7 @@
static int check_number_printing(void) { static int check_number_printing(void) {
struct { struct {
calculated_number n; NETDATA_DOUBLE n;
const char *correct; const char *correct;
} values[] = { } values[] = {
{ .n = 0, .correct = "0" }, { .n = 0, .correct = "0" },
@ -22,8 +22,8 @@ static int check_number_printing(void) {
char netdata[50], system[50]; char netdata[50], system[50];
int i, failed = 0; int i, failed = 0;
for(i = 0; values[i].correct ; i++) { for(i = 0; values[i].correct ; i++) {
print_calculated_number(netdata, values[i].n); print_netdata_double(netdata, values[i].n);
snprintfz(system, 49, "%0.12" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)values[i].n); snprintfz(system, 49, "%0.12" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)values[i].n);
int ok = 1; int ok = 1;
if(strcmp(netdata, values[i].correct) != 0) { if(strcmp(netdata, values[i].correct) != 0) {
@ -95,35 +95,36 @@ static int check_rrdcalc_comparisons(void) {
return 0; return 0;
} }
int check_storage_number(calculated_number n, int debug) { int check_storage_number(NETDATA_DOUBLE n, int debug) {
char buffer[100]; char buffer[100];
uint32_t flags = SN_DEFAULT_FLAGS; uint32_t flags = SN_DEFAULT_FLAGS;
storage_number s = pack_storage_number(n, flags); storage_number s = pack_storage_number(n, flags);
calculated_number d = unpack_storage_number(s); NETDATA_DOUBLE d = unpack_storage_number(s);
if(!does_storage_number_exist(s)) { if(!does_storage_number_exist(s)) {
fprintf(stderr, "Exists flags missing for number " CALCULATED_NUMBER_FORMAT "!\n", n); fprintf(stderr, "Exists flags missing for number " NETDATA_DOUBLE_FORMAT "!\n", n);
return 5; return 5;
} }
calculated_number ddiff = d - n; NETDATA_DOUBLE ddiff = d - n;
calculated_number dcdiff = ddiff * 100.0 / n; NETDATA_DOUBLE dcdiff = ddiff * 100.0 / n;
if(dcdiff < 0) dcdiff = -dcdiff; if(dcdiff < 0) dcdiff = -dcdiff;
size_t len = (size_t)print_calculated_number(buffer, d); size_t len = (size_t)print_netdata_double(buffer, d);
calculated_number p = str2ld(buffer, NULL); NETDATA_DOUBLE p = str2ndd(buffer, NULL);
calculated_number pdiff = n - p; NETDATA_DOUBLE pdiff = n - p;
calculated_number pcdiff = pdiff * 100.0 / n; NETDATA_DOUBLE pcdiff = pdiff * 100.0 / n;
if(pcdiff < 0) pcdiff = -pcdiff; if(pcdiff < 0) pcdiff = -pcdiff;
if(debug) { if(debug) {
fprintf(stderr, fprintf(stderr,
CALCULATED_NUMBER_FORMAT " original\n" NETDATA_DOUBLE_FORMAT
CALCULATED_NUMBER_FORMAT " packed and unpacked, (stored as 0x%08X, diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n" " original\n" NETDATA_DOUBLE_FORMAT " packed and unpacked, (stored as 0x%08X, diff " NETDATA_DOUBLE_FORMAT
"%s printed after unpacked (%zu bytes)\n" ", " NETDATA_DOUBLE_FORMAT "%%)\n"
CALCULATED_NUMBER_FORMAT " re-parsed from printed (diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n\n", "%s printed after unpacked (%zu bytes)\n" NETDATA_DOUBLE_FORMAT
" re-parsed from printed (diff " NETDATA_DOUBLE_FORMAT ", " NETDATA_DOUBLE_FORMAT "%%)\n\n",
n, n,
d, s, ddiff, dcdiff, d, s, ddiff, dcdiff,
buffer, len, buffer, len,
@ -132,10 +133,11 @@ int check_storage_number(calculated_number n, int debug) {
if(len != strlen(buffer)) fprintf(stderr, "ERROR: printed number %s is reported to have length %zu but it has %zu\n", buffer, len, strlen(buffer)); if(len != strlen(buffer)) fprintf(stderr, "ERROR: printed number %s is reported to have length %zu but it has %zu\n", buffer, len, strlen(buffer));
if(dcdiff > ACCURACY_LOSS_ACCEPTED_PERCENT) if(dcdiff > ACCURACY_LOSS_ACCEPTED_PERCENT)
fprintf(stderr, "WARNING: packing number " CALCULATED_NUMBER_FORMAT " has accuracy loss " CALCULATED_NUMBER_FORMAT " %%\n", n, dcdiff); fprintf(stderr, "WARNING: packing number " NETDATA_DOUBLE_FORMAT " has accuracy loss " NETDATA_DOUBLE_FORMAT " %%\n", n, dcdiff);
if(pcdiff > ACCURACY_LOSS_ACCEPTED_PERCENT) if(pcdiff > ACCURACY_LOSS_ACCEPTED_PERCENT)
fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " CALCULATED_NUMBER_FORMAT " has accuracy loss " CALCULATED_NUMBER_FORMAT " %%\n", n, pcdiff); fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " NETDATA_DOUBLE_FORMAT
" has accuracy loss " NETDATA_DOUBLE_FORMAT " %%\n", n, pcdiff);
} }
if(len != strlen(buffer)) return 1; if(len != strlen(buffer)) return 1;
@ -144,8 +146,8 @@ int check_storage_number(calculated_number n, int debug) {
return 0; return 0;
} }
calculated_number storage_number_min(calculated_number n) { NETDATA_DOUBLE storage_number_min(NETDATA_DOUBLE n) {
calculated_number r = 1, last; NETDATA_DOUBLE r = 1, last;
do { do {
last = n; last = n;
@ -159,12 +161,12 @@ calculated_number storage_number_min(calculated_number n) {
void benchmark_storage_number(int loop, int multiplier) { void benchmark_storage_number(int loop, int multiplier) {
int i, j; int i, j;
calculated_number n, d; NETDATA_DOUBLE n, d;
storage_number s; storage_number s;
unsigned long long user, system, total, mine, their; unsigned long long user, system, total, mine, their;
calculated_number storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW); NETDATA_DOUBLE storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW);
calculated_number storage_number_positive_max = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW); NETDATA_DOUBLE storage_number_positive_max = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW);
char buffer[100]; char buffer[100];
@ -174,25 +176,25 @@ void benchmark_storage_number(int loop, int multiplier) {
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
fprintf(stderr, "SYSTEM LONG DOUBLE SIZE: %zu bytes\n", sizeof(calculated_number)); fprintf(stderr, "SYSTEM LONG DOUBLE SIZE: %zu bytes\n", sizeof(NETDATA_DOUBLE));
fprintf(stderr, "NETDATA FLOATING POINT SIZE: %zu bytes\n", sizeof(storage_number)); fprintf(stderr, "NETDATA FLOATING POINT SIZE: %zu bytes\n", sizeof(storage_number));
mine = (calculated_number)sizeof(storage_number) * (calculated_number)loop; mine = (NETDATA_DOUBLE)sizeof(storage_number) * (NETDATA_DOUBLE)loop;
their = (calculated_number)sizeof(calculated_number) * (calculated_number)loop; their = (NETDATA_DOUBLE)sizeof(NETDATA_DOUBLE) * (NETDATA_DOUBLE)loop;
if(mine > their) { if(mine > their) {
fprintf(stderr, "\nNETDATA NEEDS %0.2" LONG_DOUBLE_MODIFIER " TIMES MORE MEMORY. Sorry!\n", (LONG_DOUBLE)(mine / their)); fprintf(stderr, "\nNETDATA NEEDS %0.2" NETDATA_DOUBLE_MODIFIER " TIMES MORE MEMORY. Sorry!\n", (NETDATA_DOUBLE)(mine / their));
} }
else { else {
fprintf(stderr, "\nNETDATA INTERNAL FLOATING POINT ARITHMETICS NEEDS %0.2" LONG_DOUBLE_MODIFIER " TIMES LESS MEMORY.\n", (LONG_DOUBLE)(their / mine)); fprintf(stderr, "\nNETDATA INTERNAL FLOATING POINT ARITHMETICS NEEDS %0.2" NETDATA_DOUBLE_MODIFIER " TIMES LESS MEMORY.\n", (NETDATA_DOUBLE)(their / mine));
} }
fprintf(stderr, "\nNETDATA FLOATING POINT\n"); fprintf(stderr, "\nNETDATA FLOATING POINT\n");
fprintf(stderr, "MIN POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW)); fprintf(stderr, "MIN POSITIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW));
fprintf(stderr, "MAX POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW)); fprintf(stderr, "MAX POSITIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW));
fprintf(stderr, "MIN NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MIN_RAW)); fprintf(stderr, "MIN NEGATIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MIN_RAW));
fprintf(stderr, "MAX NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW)); fprintf(stderr, "MAX NEGATIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW));
fprintf(stderr, "Maximum accuracy loss accepted: " CALCULATED_NUMBER_FORMAT "%%\n\n\n", (calculated_number)ACCURACY_LOSS_ACCEPTED_PERCENT); fprintf(stderr, "Maximum accuracy loss accepted: " NETDATA_DOUBLE_FORMAT "%%\n\n\n", (NETDATA_DOUBLE)ACCURACY_LOSS_ACCEPTED_PERCENT);
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
@ -207,7 +209,7 @@ void benchmark_storage_number(int loop, int multiplier) {
n *= multiplier; n *= multiplier;
if(n > storage_number_positive_max) n = storage_number_positive_min; if(n > storage_number_positive_max) n = storage_number_positive_min;
print_calculated_number(buffer, n); print_netdata_double(buffer, n);
} }
} }
@ -217,7 +219,8 @@ void benchmark_storage_number(int loop, int multiplier) {
total = user + system; total = user + system;
mine = total; mine = total;
fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0)); fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER
", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0));
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
@ -231,7 +234,7 @@ void benchmark_storage_number(int loop, int multiplier) {
for(i = 0; i < loop ;i++) { for(i = 0; i < loop ;i++) {
n *= multiplier; n *= multiplier;
if(n > storage_number_positive_max) n = storage_number_positive_min; if(n > storage_number_positive_max) n = storage_number_positive_min;
snprintfz(buffer, 100, CALCULATED_NUMBER_FORMAT, n); snprintfz(buffer, 100, NETDATA_DOUBLE_FORMAT, n);
} }
} }
@ -241,13 +244,14 @@ void benchmark_storage_number(int loop, int multiplier) {
total = user + system; total = user + system;
their = total; their = total;
fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER ", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0)); fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER
", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0));
if(mine > total) { if(mine > total) {
fprintf(stderr, "NETDATA CODE IS SLOWER %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(mine * 100.0 / their - 100.0)); fprintf(stderr, "NETDATA CODE IS SLOWER %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(mine * 100.0 / their - 100.0));
} }
else { else {
fprintf(stderr, "NETDATA CODE IS F A S T E R %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(their * 100.0 / mine - 100.0)); fprintf(stderr, "NETDATA CODE IS F A S T E R %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(their * 100.0 / mine - 100.0));
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
@ -265,7 +269,7 @@ void benchmark_storage_number(int loop, int multiplier) {
s = pack_storage_number(n, SN_DEFAULT_FLAGS); s = pack_storage_number(n, SN_DEFAULT_FLAGS);
d = unpack_storage_number(s); d = unpack_storage_number(s);
print_calculated_number(buffer, d); print_netdata_double(buffer, d);
} }
} }
@ -275,13 +279,14 @@ void benchmark_storage_number(int loop, int multiplier) {
total = user + system; total = user + system;
mine = total; mine = total;
fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER ", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0)); fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER
", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0));
if(mine > their) { if(mine > their) {
fprintf(stderr, "WITH PACKING UNPACKING NETDATA CODE IS SLOWER %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(mine * 100.0 / their - 100.0)); fprintf(stderr, "WITH PACKING UNPACKING NETDATA CODE IS SLOWER %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(mine * 100.0 / their - 100.0));
} }
else { else {
fprintf(stderr, "EVEN WITH PACKING AND UNPACKING, NETDATA CODE IS F A S T E R %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(their * 100.0 / mine - 100.0)); fprintf(stderr, "EVEN WITH PACKING AND UNPACKING, NETDATA CODE IS F A S T E R %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(their * 100.0 / mine - 100.0));
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
@ -290,13 +295,13 @@ void benchmark_storage_number(int loop, int multiplier) {
static int check_storage_number_exists() { static int check_storage_number_exists() {
uint32_t flags = SN_DEFAULT_FLAGS; uint32_t flags = SN_DEFAULT_FLAGS;
calculated_number n = 0.0; NETDATA_DOUBLE n = 0.0;
storage_number s = pack_storage_number(n, flags); storage_number s = pack_storage_number(n, flags);
calculated_number d = unpack_storage_number(s); NETDATA_DOUBLE d = unpack_storage_number(s);
if(n != d) { if(n != d) {
fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d); fprintf(stderr, "Wrong number returned. Expected " NETDATA_DOUBLE_FORMAT ", returned " NETDATA_DOUBLE_FORMAT "!\n", n, d);
return 1; return 1;
} }
@ -306,10 +311,10 @@ static int check_storage_number_exists() {
int unit_test_storage() { int unit_test_storage() {
if(check_storage_number_exists()) return 0; if(check_storage_number_exists()) return 0;
calculated_number storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW); NETDATA_DOUBLE storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW);
calculated_number storage_number_negative_max = unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW); NETDATA_DOUBLE storage_number_negative_max = unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW);
calculated_number c, a = 0; NETDATA_DOUBLE c, a = 0;
int i, j, g, r = 0; int i, j, g, r = 0;
for(g = -1; g <= 1 ; g++) { for(g = -1; g <= 1 ; g++) {
@ -343,23 +348,26 @@ int unit_test_str2ld() {
int i; int i;
for(i = 0; values[i] ; i++) { for(i = 0; values[i] ; i++) {
char *e_mine = "hello", *e_sys = "world"; char *e_mine = "hello", *e_sys = "world";
LONG_DOUBLE mine = str2ld(values[i], &e_mine); NETDATA_DOUBLE mine = str2ndd(values[i], &e_mine);
LONG_DOUBLE sys = strtold(values[i], &e_sys); NETDATA_DOUBLE sys = strtondd(values[i], &e_sys);
if(isnan(mine)) { if(isnan(mine)) {
if(!isnan(sys)) { if(!isnan(sys)) {
fprintf(stderr, "Value '%s' is parsed as %" LONG_DOUBLE_MODIFIER ", but system believes it is %" LONG_DOUBLE_MODIFIER ".\n", values[i], mine, sys); fprintf(stderr, "Value '%s' is parsed as %" NETDATA_DOUBLE_MODIFIER
", but system believes it is %" NETDATA_DOUBLE_MODIFIER ".\n", values[i], mine, sys);
return -1; return -1;
} }
} }
else if(isinf(mine)) { else if(isinf(mine)) {
if(!isinf(sys)) { if(!isinf(sys)) {
fprintf(stderr, "Value '%s' is parsed as %" LONG_DOUBLE_MODIFIER ", but system believes it is %" LONG_DOUBLE_MODIFIER ".\n", values[i], mine, sys); fprintf(stderr, "Value '%s' is parsed as %" NETDATA_DOUBLE_MODIFIER
", but system believes it is %" NETDATA_DOUBLE_MODIFIER ".\n", values[i], mine, sys);
return -1; return -1;
} }
} }
else if(mine != sys && ABS(mine-sys) > 0.000001) { else if(mine != sys && ABS(mine-sys) > 0.000001) {
fprintf(stderr, "Value '%s' is parsed as %" LONG_DOUBLE_MODIFIER ", but system believes it is %" LONG_DOUBLE_MODIFIER ", delta %" LONG_DOUBLE_MODIFIER ".\n", values[i], mine, sys, sys-mine); fprintf(stderr, "Value '%s' is parsed as %" NETDATA_DOUBLE_MODIFIER
", but system believes it is %" NETDATA_DOUBLE_MODIFIER ", delta %" NETDATA_DOUBLE_MODIFIER ".\n", values[i], mine, sys, sys-mine);
return -1; return -1;
} }
@ -368,7 +376,8 @@ int unit_test_str2ld() {
return -1; return -1;
} }
fprintf(stderr, "str2ld() parsed value '%s' exactly the same way with strtold(), returned %" LONG_DOUBLE_MODIFIER " vs %" LONG_DOUBLE_MODIFIER "\n", values[i], mine, sys); fprintf(stderr, "str2ndd() parsed value '%s' exactly the same way with strtold(), returned %" NETDATA_DOUBLE_MODIFIER
" vs %" NETDATA_DOUBLE_MODIFIER "\n", values[i], mine, sys);
} }
return 0; return 0;
@ -461,10 +470,10 @@ struct test {
unsigned long feed_entries; unsigned long feed_entries;
unsigned long result_entries; unsigned long result_entries;
struct feed_values *feed; struct feed_values *feed;
calculated_number *results; NETDATA_DOUBLE *results;
collected_number *feed2; collected_number *feed2;
calculated_number *results2; NETDATA_DOUBLE *results2;
}; };
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
@ -484,7 +493,7 @@ struct feed_values test1_feed[] = {
{ 1000000, 100 }, { 1000000, 100 },
}; };
calculated_number test1_results[] = { NETDATA_DOUBLE test1_results[] = {
20, 30, 40, 50, 60, 70, 80, 90, 100 20, 30, 40, 50, 60, 70, 80, 90, 100
}; };
@ -520,7 +529,7 @@ struct feed_values test2_feed[] = {
{ 1000000, 100 }, { 1000000, 100 },
}; };
calculated_number test2_results[] = { NETDATA_DOUBLE test2_results[] = {
20, 30, 40, 50, 60, 70, 80, 90, 100 20, 30, 40, 50, 60, 70, 80, 90, 100
}; };
@ -555,7 +564,7 @@ struct feed_values test3_feed[] = {
{ 1000000, 100 }, { 1000000, 100 },
}; };
calculated_number test3_results[] = { NETDATA_DOUBLE test3_results[] = {
10, 10, 10, 10, 10, 10, 10, 10, 10 10, 10, 10, 10, 10, 10, 10, 10, 10
}; };
@ -590,7 +599,7 @@ struct feed_values test4_feed[] = {
{ 1000000, 100 }, { 1000000, 100 },
}; };
calculated_number test4_results[] = { NETDATA_DOUBLE test4_results[] = {
10, 10, 10, 10, 10, 10, 10, 10, 10 10, 10, 10, 10, 10, 10, 10, 10, 10
}; };
@ -625,7 +634,7 @@ struct feed_values test5_feed[] = {
{ 1000000, 0x00000000FFFFFFFFULL / 15 * 0 }, { 1000000, 0x00000000FFFFFFFFULL / 15 * 0 },
}; };
calculated_number test5_results[] = { NETDATA_DOUBLE test5_results[] = {
0x00000000FFFFFFFFULL / 15 * 7, 0x00000000FFFFFFFFULL / 15 * 7,
0x00000000FFFFFFFFULL / 15 * 7, 0x00000000FFFFFFFFULL / 15 * 7,
0x00000000FFFFFFFFULL / 15, 0x00000000FFFFFFFFULL / 15,
@ -668,7 +677,7 @@ struct feed_values test5b_feed[] = {
{ 1000000, 0xFFFFFFFFFFFFFFFFULL / 15 * 0 }, { 1000000, 0xFFFFFFFFFFFFFFFFULL / 15 * 0 },
}; };
calculated_number test5b_results[] = { NETDATA_DOUBLE test5b_results[] = {
0xFFFFFFFFFFFFFFFFULL / 15 * 7, 0xFFFFFFFFFFFFFFFFULL / 15 * 7,
0xFFFFFFFFFFFFFFFFULL / 15 * 7, 0xFFFFFFFFFFFFFFFFULL / 15 * 7,
0xFFFFFFFFFFFFFFFFULL / 15, 0xFFFFFFFFFFFFFFFFULL / 15,
@ -717,7 +726,7 @@ struct feed_values test6_feed[] = {
{ 250000, 16000 }, { 250000, 16000 },
}; };
calculated_number test6_results[] = { NETDATA_DOUBLE test6_results[] = {
4000, 4000, 4000, 4000 4000, 4000, 4000, 4000
}; };
@ -752,7 +761,7 @@ struct feed_values test7_feed[] = {
{ 2000000, 10000 }, { 2000000, 10000 },
}; };
calculated_number test7_results[] = { NETDATA_DOUBLE test7_results[] = {
500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500
}; };
@ -783,7 +792,7 @@ struct feed_values test8_feed[] = {
{ 2000000, 6000 }, { 2000000, 6000 },
}; };
calculated_number test8_results[] = { NETDATA_DOUBLE test8_results[] = {
1250, 2000, 2250, 3000, 3250, 4000, 4250, 5000, 5250, 6000 1250, 2000, 2250, 3000, 3250, 4000, 4250, 5000, 5250, 6000
}; };
@ -824,7 +833,7 @@ struct feed_values test9_feed[] = {
{ 250000, 16000 }, { 250000, 16000 },
}; };
calculated_number test9_results[] = { NETDATA_DOUBLE test9_results[] = {
4000, 8000, 12000, 16000 4000, 8000, 12000, 16000
}; };
@ -859,7 +868,7 @@ struct feed_values test10_feed[] = {
{ 1000000, 6900 + 1000 }, { 1000000, 6900 + 1000 },
}; };
calculated_number test10_results[] = { NETDATA_DOUBLE test10_results[] = {
1000, 1000, 1000, 1000, 1000, 1000, 1000 1000, 1000, 1000, 1000, 1000, 1000, 1000
}; };
@ -898,11 +907,11 @@ collected_number test11_feed2[] = {
10, 20, 30, 40, 50, 60, 70, 80, 90, 100 10, 20, 30, 40, 50, 60, 70, 80, 90, 100
}; };
calculated_number test11_results[] = { NETDATA_DOUBLE test11_results[] = {
50, 50, 50, 50, 50, 50, 50, 50, 50 50, 50, 50, 50, 50, 50, 50, 50, 50
}; };
calculated_number test11_results2[] = { NETDATA_DOUBLE test11_results2[] = {
50, 50, 50, 50, 50, 50, 50, 50, 50 50, 50, 50, 50, 50, 50, 50, 50, 50
}; };
@ -941,11 +950,11 @@ collected_number test12_feed2[] = {
10*3, 20*3, 30*3, 40*3, 50*3, 60*3, 70*3, 80*3, 90*3, 100*3 10*3, 20*3, 30*3, 40*3, 50*3, 60*3, 70*3, 80*3, 90*3, 100*3
}; };
calculated_number test12_results[] = { NETDATA_DOUBLE test12_results[] = {
25, 25, 25, 25, 25, 25, 25, 25, 25 25, 25, 25, 25, 25, 25, 25, 25, 25
}; };
calculated_number test12_results2[] = { NETDATA_DOUBLE test12_results2[] = {
75, 75, 75, 75, 75, 75, 75, 75, 75 75, 75, 75, 75, 75, 75, 75, 75, 75
}; };
@ -980,7 +989,7 @@ struct feed_values test13_feed[] = {
{ 1000000, 6900 + 1000 }, { 1000000, 6900 + 1000 },
}; };
calculated_number test13_results[] = { NETDATA_DOUBLE test13_results[] = {
83.3333300, 100, 100, 100, 100, 100, 100 83.3333300, 100, 100, 100, 100, 100, 100
}; };
@ -1015,7 +1024,7 @@ struct feed_values test14_feed[] = {
{ 29942000, 0x0153987f888982d0ULL }, { 29942000, 0x0153987f888982d0ULL },
}; };
calculated_number test14_results[] = { NETDATA_DOUBLE test14_results[] = {
23.1383300, 21.8515600, 21.8804600, 21.7788000, 22.0112200, 22.4386100, 22.0906100, 21.9150800 23.1383300, 21.8515600, 21.8804600, 21.7788000, 22.0112200, 22.4386100, 22.0906100, 21.9150800
}; };
@ -1047,7 +1056,7 @@ struct feed_values test14b_feed[] = {
{ 29942000, 13573000 + 29969000 + 29958000 + 30054000 + 34952000 + 25046000 + 29947000 + 30054000 + 29942000 }, { 29942000, 13573000 + 29969000 + 29958000 + 30054000 + 34952000 + 25046000 + 29947000 + 30054000 + 29942000 },
}; };
calculated_number test14b_results[] = { NETDATA_DOUBLE test14b_results[] = {
1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000
}; };
@ -1079,7 +1088,7 @@ struct feed_values test14c_feed[] = {
{ 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 }, { 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 },
}; };
calculated_number test14c_results[] = { NETDATA_DOUBLE test14c_results[] = {
1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000
}; };
@ -1118,11 +1127,11 @@ collected_number test15_feed2[] = {
178825286, 178825286, 178825286, 178825286, 178825498, 178825498, 179165652, 179202964, 179203282, 179204130 178825286, 178825286, 178825286, 178825286, 178825498, 178825498, 179165652, 179202964, 179203282, 179204130
}; };
calculated_number test15_results[] = { NETDATA_DOUBLE test15_results[] = {
5857.4080000, 5898.4540000, 5891.6590000, 5806.3160000, 5914.2640000, 3202.2630000, 5589.6560000, 5822.5260000, 5911.7520000 5857.4080000, 5898.4540000, 5891.6590000, 5806.3160000, 5914.2640000, 3202.2630000, 5589.6560000, 5822.5260000, 5911.7520000
}; };
calculated_number test15_results2[] = { NETDATA_DOUBLE test15_results2[] = {
0.0000000, 0.0000000, 0.0024944, 1.6324779, 0.0212777, 2655.1890000, 290.5387000, 5.6733610, 6.5960220 0.0000000, 0.0000000, 0.0024944, 1.6324779, 0.0212777, 2655.1890000, 290.5387000, 5.6733610, 6.5960220
}; };
@ -1173,12 +1182,13 @@ int run_test(struct test *test)
if(c) { if(c) {
time_now += test->feed[c].microseconds; time_now += test->feed[c].microseconds;
fprintf(stderr, " > %s: feeding position %lu, after %0.3f seconds (%0.3f seconds from start), delta " CALCULATED_NUMBER_FORMAT ", rate " CALCULATED_NUMBER_FORMAT "\n", fprintf(stderr, " > %s: feeding position %lu, after %0.3f seconds (%0.3f seconds from start), delta " NETDATA_DOUBLE_FORMAT
", rate " NETDATA_DOUBLE_FORMAT "\n",
test->name, c+1, test->name, c+1,
(float)test->feed[c].microseconds / 1000000.0, (float)test->feed[c].microseconds / 1000000.0,
(float)time_now / 1000000.0, (float)time_now / 1000000.0,
((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor, ((NETDATA_DOUBLE)test->feed[c].value - (NETDATA_DOUBLE)last) * (NETDATA_DOUBLE)test->multiplier / (NETDATA_DOUBLE)test->divisor,
(((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor) / (calculated_number)test->feed[c].microseconds * (calculated_number)1000000); (((NETDATA_DOUBLE)test->feed[c].value - (NETDATA_DOUBLE)last) * (NETDATA_DOUBLE)test->multiplier / (NETDATA_DOUBLE)test->divisor) / (NETDATA_DOUBLE)test->feed[c].microseconds * (NETDATA_DOUBLE)1000000);
// rrdset_next_usec_unfiltered(st, test->feed[c].microseconds); // rrdset_next_usec_unfiltered(st, test->feed[c].microseconds);
st->usec_since_last_update = test->feed[c].microseconds; st->usec_since_last_update = test->feed[c].microseconds;
@ -1216,10 +1226,11 @@ int run_test(struct test *test)
unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries; unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries;
for(c = 0 ; c < max ; c++) { for(c = 0 ; c < max ; c++) {
calculated_number v = unpack_storage_number(rd->values[c]); NETDATA_DOUBLE v = unpack_storage_number(rd->db[c]);
calculated_number n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS)); NETDATA_DOUBLE n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS));
int same = (calculated_number_round(v * 10000000.0) == calculated_number_round(n * 10000000.0))?1:0; int same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n", fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", %s\n",
test->name, rd->name, c+1, test->name, rd->name, c+1,
(int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start), (int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start),
n, v, (same)?"OK":"### E R R O R ###"); n, v, (same)?"OK":"### E R R O R ###");
@ -1227,10 +1238,11 @@ int run_test(struct test *test)
if(!same) errors++; if(!same) errors++;
if(rd2) { if(rd2) {
v = unpack_storage_number(rd2->values[c]); v = unpack_storage_number(rd2->db[c]);
n = test->results2[c]; n = test->results2[c];
same = (calculated_number_round(v * 10000000.0) == calculated_number_round(n * 10000000.0))?1:0; same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n", fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", %s\n",
test->name, rd2->name, c+1, test->name, rd2->name, c+1,
(int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start), (int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start),
n, v, (same)?"OK":"### E R R O R ###"); n, v, (same)?"OK":"### E R R O R ###");
@ -1470,14 +1482,14 @@ int unit_test(long delay, long shift)
int ret = 0; int ret = 0;
storage_number sn; storage_number sn;
calculated_number cn, v; NETDATA_DOUBLE cn, v;
for(c = 0 ; c < st->counter ; c++) { for(c = 0 ; c < st->counter ; c++) {
fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10); fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10);
for(rd = st->dimensions ; rd ; rd = rd->next) { for(rd = st->dimensions ; rd ; rd = rd->next) {
sn = rd->values[c]; sn = rd->db[c];
cn = unpack_storage_number(sn); cn = unpack_storage_number(sn);
fprintf(stderr, "\t %s " CALCULATED_NUMBER_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rd->id, cn, sn); fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rd->id, cn, sn);
if(rd == rdabs) v = if(rd == rdabs) v =
( oincrement ( oincrement
@ -1492,7 +1504,7 @@ int unit_test(long delay, long shift)
if(v == cn) fprintf(stderr, "passed.\n"); if(v == cn) fprintf(stderr, "passed.\n");
else { else {
fprintf(stderr, "ERROR! (expected " CALCULATED_NUMBER_FORMAT ")\n", v); fprintf(stderr, "ERROR! (expected " NETDATA_DOUBLE_FORMAT ")\n", v);
ret = 1; ret = 1;
} }
} }
@ -1742,7 +1754,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
time_t time_now, time_retrieved; time_t time_now, time_retrieved;
int i, j, k, c, errors, update_every; int i, j, k, c, errors, update_every;
collected_number last; collected_number last;
calculated_number value, expected; NETDATA_DOUBLE value, expected;
SN_FLAGS nflags; SN_FLAGS nflags;
struct rrddim_query_handle handle; struct rrddim_query_handle handle;
size_t value_errors = 0, time_errors = 0; size_t value_errors = 0, time_errors = 0;
@ -1759,16 +1771,16 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
for (k = 0; k < QUERY_BATCH; ++k) { for (k = 0; k < QUERY_BATCH; ++k) {
last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] + last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
j * REGION_POINTS[current_region] + c + k; j * REGION_POINTS[current_region] + c + k;
expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
time_t end_time; time_t end_time;
value = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags); value = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags);
same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
if(!same) { if(!same) {
if(!value_errors) if(!value_errors)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", ", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, expected, value); st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, expected, value);
value_errors++; value_errors++;
errors++; errors++;
@ -1806,7 +1818,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
int i, j, errors, value_errors = 0, time_errors = 0; int i, j, errors, value_errors = 0, time_errors = 0;
long c; long c;
collected_number last; collected_number last;
calculated_number value, expected; NETDATA_DOUBLE value, expected;
errors = 0; errors = 0;
long points = (time_end - time_start) / update_every; long points = (time_end - time_start) / update_every;
@ -1825,18 +1837,18 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
// for each dimension // for each dimension
for (j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) { for (j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) {
calculated_number *cn = &r->v[ c * r->d ]; NETDATA_DOUBLE *cn = &r->v[ c * r->d ];
value = cn[j]; value = cn[j];
assert(rd[i][j] == d); assert(rd[i][j] == d);
last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c; last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c;
expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
if(!same) { if(!same) {
if(value_errors < 10) if(value_errors < 10)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
CALCULATED_NUMBER_FORMAT ", RRDR found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", ", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value); st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
value_errors++; value_errors++;
} }
@ -1959,18 +1971,18 @@ int test_dbengine(void)
// for each dimension // for each dimension
for(j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) { for(j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) {
calculated_number *cn = &r->v[ c * r->d ]; NETDATA_DOUBLE *cn = &r->v[ c * r->d ];
calculated_number value = cn[j]; NETDATA_DOUBLE value = cn[j];
assert(rd[i][j] == d); assert(rd[i][j] == d);
collected_number last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c - point_offset + 1; collected_number last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c - point_offset + 1;
calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); NETDATA_DOUBLE expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
uint8_t same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; uint8_t same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
if(!same) { if(!same) {
if(!value_errors) if(!value_errors)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
CALCULATED_NUMBER_FORMAT ", RRDR found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", ", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value); st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
value_errors++; value_errors++;
} }
@ -2167,7 +2179,7 @@ static void query_dbengine_chart(void *arg)
uint8_t same; uint8_t same;
time_t time_now, time_retrieved; time_t time_now, time_retrieved;
collected_number generatedv; collected_number generatedv;
calculated_number value, expected; NETDATA_DOUBLE value, expected;
SN_FLAGS nflags; SN_FLAGS nflags;
struct rrddim_query_handle handle; struct rrddim_query_handle handle;
size_t value_errors = 0, time_errors = 0; size_t value_errors = 0, time_errors = 0;
@ -2200,12 +2212,12 @@ static void query_dbengine_chart(void *arg)
++thread_info->queries_nr; ++thread_info->queries_nr;
for (time_now = time_after ; time_now <= time_before ; time_now += update_every) { for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
generatedv = generate_dbengine_chart_value(i, j, time_now); generatedv = generate_dbengine_chart_value(i, j, time_now);
expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_DEFAULT_FLAGS)); expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE) generatedv, SN_DEFAULT_FLAGS));
if (unlikely(rd->state->query_ops.is_finished(&handle))) { if (unlikely(rd->state->query_ops.is_finished(&handle))) {
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n", ", found data gap, ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected); st->name, rd->name, (unsigned long) time_now, expected);
++thread_info->errors; ++thread_info->errors;
} }
@ -2213,10 +2225,10 @@ static void query_dbengine_chart(void *arg)
} }
time_t end_time; time_t end_time;
value = rd->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags); value = rd->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags);
if (!calculated_number_isnumber(value)) { if (!netdata_double_isnumber(value)) {
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n", ", found data gap, ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected); st->name, rd->name, (unsigned long) time_now, expected);
++thread_info->errors; ++thread_info->errors;
} }
@ -2224,13 +2236,12 @@ static void query_dbengine_chart(void *arg)
} }
++thread_info->queried_metrics_nr; ++thread_info->queried_metrics_nr;
same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
if (!same) { if (!same) {
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
if(!value_errors) if(!value_errors)
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
", ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected, value); st->name, rd->name, (unsigned long) time_now, expected, value);
value_errors++; value_errors++;
thread_info->errors++; thread_info->errors++;

41
database/README.md
View File

@ -13,7 +13,7 @@ Netdata is fully capable of long-term metrics storage, at per-second granularity
traditional database. There is some amount of RAM dedicated to data caching and indexing and the rest of the data traditional database. There is some amount of RAM dedicated to data caching and indexing and the rest of the data
reside compressed on disk. The number of history entries is not fixed in this case, but depends on the configured reside compressed on disk. The number of history entries is not fixed in this case, but depends on the configured
disk space and the effective compression ratio of the data stored. This is the **only mode** that supports changing disk space and the effective compression ratio of the data stored. This is the **only mode** that supports changing
the data collection update frequency (`update_every`) **without losing** the previously stored metrics. For more the data collection update frequency (`update every`) **without losing** the previously stored metrics. For more
details see [here](/database/engine/README.md). details see [here](/database/engine/README.md).
2. `ram`, data are purely in memory. Data are never saved on disk. This mode uses `mmap()` and supports [KSM](#ksm). 2. `ram`, data are purely in memory. Data are never saved on disk. This mode uses `mmap()` and supports [KSM](#ksm).
@ -32,15 +32,12 @@ Netdata is fully capable of long-term metrics storage, at per-second granularity
6. `alloc`, like `ram` but it uses `calloc()` and does not support [KSM](#ksm). This mode is the fallback for all 6. `alloc`, like `ram` but it uses `calloc()` and does not support [KSM](#ksm). This mode is the fallback for all
others except `none`. others except `none`.
You can select the memory mode by editing `netdata.conf` and setting: You can select the database mode by editing `netdata.conf` and setting:
```conf ```conf
[global] [db]
# dbengine (default), ram, save (the default if dbengine not available), map (swap like), none, alloc # dbengine (default), ram, save (the default if dbengine not available), map (swap like), none, alloc
memory mode = dbengine mode = dbengine
# the directory where data are saved
cache directory = /var/cache/netdata
``` ```
## Running Netdata in embedded devices ## Running Netdata in embedded devices
@ -49,26 +46,23 @@ Embedded devices usually have very limited RAM resources available.
There are 2 settings for you to tweak: There are 2 settings for you to tweak:
1. `update every`, which controls the data collection frequency 1. `[db].update every`, which controls the data collection frequency
2. `history`, which controls the size of the database in RAM (except for `memory mode = dbengine`) 2. `[db].retention`, which controls the size of the database in memory (except for `[db].mode = dbengine`)
By default `update every = 1` and `history = 3600`. This gives you an hour of data with per second updates. By default `[db].update every = 1` and `[db].retention = 3600`. This gives you an hour of data with per second updates.
If you set `update every = 2` and `history = 1800`, you will still have an hour of data, but collected once every 2 If you set `[db].update every = 2` and `[db].retention = 1800`, you will still have an hour of data, but collected once every 2
seconds. This will **cut in half** both CPU and RAM resources consumed by Netdata. Of course experiment a bit. On very seconds. This will **cut in half** both CPU and RAM resources consumed by Netdata. Of course experiment a bit. On very
weak devices you might have to use `update every = 5` and `history = 720` (still 1 hour of data, but 1/5 of the CPU and weak devices you might have to use `[db].update every = 5` and `[db].retention = 720` (still 1 hour of data, but 1/5 of the CPU and
RAM resources). RAM resources).
You can also disable [data collection plugins](/collectors/README.md) you don't need. Disabling such plugins will also free both You can also disable [data collection plugins](/collectors/README.md) you don't need. Disabling such plugins will also free both
CPU and RAM resources. CPU and RAM resources.
## Running a dedicated central Netdata server ## Running a dedicated parent Netdata server
Netdata allows streaming data between Netdata nodes. This allows us to have a central Netdata server that will maintain Netdata allows streaming data between Netdata nodes in real-time. This allows having one or more parent Netdata servers that will maintain
the entire database for all nodes, and will also run health checks/alarms for all nodes. the entire database for all the nodes that connect to them (their children), and will also run health checks/alarms for all these nodes.
For this central Netdata, memory size can be a problem. Fortunately, Netdata supports several memory modes. **One
interesting option** for this setup is `memory mode = map`.
### map ### map
@ -77,8 +71,7 @@ in memory, but the kernel automatically loads and saves memory pages from/to dis
**We suggest _not_ to use this mode on nodes that run other applications.** There will always be dirty memory to be **We suggest _not_ to use this mode on nodes that run other applications.** There will always be dirty memory to be
synced and this syncing process may influence the way other applications work. This mode however is useful when we need synced and this syncing process may influence the way other applications work. This mode however is useful when we need
a central Netdata server that would normally need huge amounts of memory. Using memory mode `map` we can overcome all a parent Netdata server that would normally need huge amounts of memory.
memory restrictions.
There are a few kernel options that provide finer control on the way this syncing works. But before explaining them, a There are a few kernel options that provide finer control on the way this syncing works. But before explaining them, a
brief introduction of how Netdata database works is needed. brief introduction of how Netdata database works is needed.
@ -142,8 +135,8 @@ vm.dirty_ratio = 90
vm.dirty_writeback_centisecs = 0 vm.dirty_writeback_centisecs = 0
``` ```
There is another memory mode to help overcome the memory size problem. What is **most interesting for this setup** is There is another mode to help overcome the memory size problem. What is **most interesting for this setup** is
`memory mode = dbengine`. `[db].mode = dbengine`.
### dbengine ### dbengine
@ -154,12 +147,12 @@ configured disk space and the effective compression ratio of the data stored.
We suggest to use **this** mode on nodes that also run other applications. The Database Engine uses direct I/O to avoid We suggest to use **this** mode on nodes that also run other applications. The Database Engine uses direct I/O to avoid
polluting the OS filesystem caches and does not generate excessive I/O traffic so as to create the minimum possible polluting the OS filesystem caches and does not generate excessive I/O traffic so as to create the minimum possible
interference with other applications. Using memory mode `dbengine` we can overcome most memory restrictions. For more interference with other applications. Using mode `dbengine` we can overcome most memory restrictions. For more
details see [here](/database/engine/README.md). details see [here](/database/engine/README.md).
## KSM ## KSM
Netdata offers all its round robin database to kernel for deduplication (except for `memory mode = dbengine`). Netdata offers all its in-memory database to kernel for deduplication.
In the past KSM has been criticized for consuming a lot of CPU resources. Although this is true when KSM is used for In the past KSM has been criticized for consuming a lot of CPU resources. Although this is true when KSM is used for
deduplicating certain applications, it is not true with netdata, since the Netdata memory is written very infrequently deduplicating certain applications, it is not true with netdata, since the Netdata memory is written very infrequently

65
database/engine/README.md
View File

@ -7,63 +7,62 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/database/engine/
# Database engine # Database engine
The Database Engine works like a traditional database. It dedicates a certain amount of RAM to data caching and The Database Engine works like a traditional database. It dedicates a certain amount of RAM to data caching and
indexing, while the rest of the data resides compressed on disk. Unlike other [memory modes](/database/README.md), the indexing, while the rest of the data resides compressed on disk. Unlike other [database modes](/database/README.md), the
amount of historical metrics stored is based on the amount of disk space you allocate and the effective compression amount of historical metrics stored is based on the amount of disk space you allocate and the effective compression
ratio, not a fixed number of metrics collected. ratio, not a fixed number of metrics collected.
By using both RAM and disk space, the database engine allows for long-term storage of per-second metrics inside of the By using both RAM and disk space, the database engine allows for long-term storage of per-second metrics inside of the
Agent itself. Agent itself.
In addition, the database engine is the only memory mode that supports changing the data collection update frequency In addition, the dbengine is the only mode that supports changing the data collection update frequency
(`update_every`) without losing the metrics your Agent already gathered and stored. (`update every`) without losing the metrics your Agent already gathered and stored.
## Configuration ## Configuration
To use the database engine, open `netdata.conf` and set `memory mode` to `dbengine`. To use the database engine, open `netdata.conf` and set `[db].mode` to `dbengine`.
```conf ```conf
[global] [db]
memory mode = dbengine mode = dbengine
``` ```
To configure the database engine, look for the `page cache size` and `dbengine multihost disk space` settings in the To configure the database engine, look for the `page cache size MB` and `dbengine multihost disk space MB` settings in the
`[global]` section of your `netdata.conf`. The Agent ignores the `history` setting when using the database engine. `[db]` section of your `netdata.conf`. The Agent ignores the `[db].retention` setting when using the dbengine.
```conf ```conf
[global] [db]
page cache size = 32 page cache size MB = 32
dbengine multihost disk space = 256 dbengine multihost disk space MB = 256
``` ```
The above values are the default values for Page Cache size and DB engine disk space quota. Both numbers are The above values are the default values for Page Cache size and DB engine disk space quota.
in **MiB**.
The `page cache size` option determines the amount of RAM in **MiB** dedicated to caching Netdata metric values. The The `page cache size MB` option determines the amount of RAM dedicated to caching Netdata metric values. The
actual page cache size will be slightly larger than this figure—see the [memory requirements](#memory-requirements) actual page cache size will be slightly larger than this figure—see the [memory requirements](#memory-requirements)
section for details. section for details.
The `dbengine multihost disk space` option determines the amount of disk space in **MiB** that is dedicated to storing The `dbengine multihost disk space MB` option determines the amount of disk space that is dedicated to storing
Netdata metric values and all related metadata describing them. You can use the [**database engine Netdata metric values and all related metadata describing them. You can use the [**database engine
calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics) calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics)
to correctly set `dbengine multihost disk space` based on your metrics retention policy. The calculator gives an to correctly set `dbengine multihost disk space MB` based on your metrics retention policy. The calculator gives an
accurate estimate based on how many child nodes you have, how many metrics your Agent collects, and more. accurate estimate based on how many child nodes you have, how many metrics your Agent collects, and more.
### Legacy configuration ### Legacy configuration
The deprecated `dbengine disk space` option determines the amount of disk space in **MiB** that is dedicated to storing The deprecated `dbengine disk space MB` option determines the amount of disk space that is dedicated to storing
Netdata metric values per legacy database engine instance (see [details on the legacy mode](#legacy-mode) below). Netdata metric values per legacy database engine instance (see [details on the legacy mode](#legacy-mode) below).
```conf ```conf
[global] [db]
dbengine disk space = 256 dbengine disk space MB = 256
``` ```
### Streaming metrics to the database engine ### Streaming metrics to the database engine
When using the multihost database engine, all parent and child nodes share the same `page cache size` and `dbengine When using the multihost database engine, all parent and child nodes share the same `page cache size MB` and `dbengine
multihost disk space` in a single dbengine instance. The [**database engine multihost disk space MB` in a single dbengine instance. The [**database engine
calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics) calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics)
helps you properly set `page cache size` and `dbengine multihost disk space` on your parent node to allocate enough helps you properly set `page cache size MB` and `dbengine multihost disk space MB` on your parent node to allocate enough
resources based on your metrics retention policy and how many child nodes you have. resources based on your metrics retention policy and how many child nodes you have.
#### Legacy mode #### Legacy mode
@ -72,8 +71,8 @@ _For Netdata Agents earlier than v1.23.2_, the Agent on the parent node uses one
another instance for every child node it receives metrics from. If you had four streaming nodes, you would have five another instance for every child node it receives metrics from. If you had four streaming nodes, you would have five
instances in total (`1 parent + 4 child nodes = 5 instances`). instances in total (`1 parent + 4 child nodes = 5 instances`).
The Agent allocates resources for each instance separately using the `dbengine disk space` (**deprecated**) setting. If The Agent allocates resources for each instance separately using the `dbengine disk space MB` (**deprecated**) setting. If
`dbengine disk space`(**deprecated**) is set to the default `256`, each instance is given 256 MiB in disk space, which `dbengine disk space MB`(**deprecated**) is set to the default `256`, each instance is given 256 MiB in disk space, which
means the total disk space required to store all instances is, roughly, `256 MiB * 1 parent * 4 child nodes = 1280 MiB`. means the total disk space required to store all instances is, roughly, `256 MiB * 1 parent * 4 child nodes = 1280 MiB`.
#### Backward compatibility #### Backward compatibility
@ -88,18 +87,18 @@ Agent.
##### Information ##### Information
For more information about setting `memory mode` on your nodes, in addition to other streaming configurations, see For more information about setting `[db].mode` on your nodes, in addition to other streaming configurations, see
[streaming](/streaming/README.md). [streaming](/streaming/README.md).
### Memory requirements ### Memory requirements
Using memory mode `dbengine` we can overcome most memory restrictions and store a dataset that is much larger than the Using database mode `dbengine` we can overcome most memory restrictions and store a dataset that is much larger than the
available memory. available memory.
There are explicit memory requirements **per** DB engine **instance**: There are explicit memory requirements **per** DB engine **instance**:
- The total page cache memory footprint will be an additional `#dimensions-being-collected x 4096 x 2` bytes over what - The total page cache memory footprint will be an additional `#dimensions-being-collected x 4096 x 2` bytes over what
the user configured with `page cache size`. the user configured with `page cache size MB`.
- an additional `#pages-on-disk x 4096 x 0.03` bytes of RAM are allocated for metadata. - an additional `#pages-on-disk x 4096 x 0.03` bytes of RAM are allocated for metadata.
@ -132,7 +131,7 @@ Netdata allocates 25% of the available file descriptors to its Database Engine i
the file descriptors that are available to the Netdata service are accessible by dbengine instances. You should take the file descriptors that are available to the Netdata service are accessible by dbengine instances. You should take
that into account when configuring your service or system-wide file descriptor limits. You can roughly estimate that the that into account when configuring your service or system-wide file descriptor limits. You can roughly estimate that the
Netdata service needs 2048 file descriptors for every 10 streaming child hosts when streaming is configured to use Netdata service needs 2048 file descriptors for every 10 streaming child hosts when streaming is configured to use
`memory mode = dbengine`. `[db].mode = dbengine`.
If for example one wants to allocate 65536 file descriptors to the Netdata service on a systemd system one needs to If for example one wants to allocate 65536 file descriptors to the Netdata service on a systemd system one needs to
override the Netdata service by running `sudo systemctl edit netdata` and creating a file with contents: override the Netdata service by running `sudo systemctl edit netdata` and creating a file with contents:
@ -166,7 +165,7 @@ You can apply the settings by running `sysctl -p` or by rebooting.
## Files ## Files
With the DB engine memory mode the metric data are stored in database files. These files are organized in pairs, the With the DB engine mode the metric data are stored in database files. These files are organized in pairs, the
datafiles and their corresponding journalfiles, e.g.: datafiles and their corresponding journalfiles, e.g.:
```sh ```sh
@ -232,10 +231,10 @@ so as to avoid all disk bottlenecks.
The reported numbers are the following: The reported numbers are the following:
| device | page cache | dataset | reads/sec | writes/sec | | device | page cache | dataset | reads/sec | writes/sec |
| :----: | :--------: | ------: | --------: | ---------: | |:------:|:----------:|--------:|----------:|-----------:|
| HDD | 64 MiB | 4.1 GiB | 813K | 18.0M | | HDD | 64 MiB | 4.1 GiB | 813K | 18.0M |
| SSD | 64 MiB | 9.8 GiB | 1.7M | 43.0M | | SSD | 64 MiB | 9.8 GiB | 1.7M | 43.0M |
| N/A | 16 GiB | 6.8 GiB | 118.2M | 30.2M | | N/A | 16 GiB | 6.8 GiB | 118.2M | 30.2M |
where "reads/sec" is the number of metric data points being read from the database via its API per second and where "reads/sec" is the number of metric data points being read from the database via its API per second and
"writes/sec" is the number of metric data points being written to the database per second. "writes/sec" is the number of metric data points being written to the database per second.

14
database/engine/rrdengine.c
View File

@ -1074,16 +1074,12 @@ struct rrdeng_cmd rrdeng_deq_cmd(struct rrdengine_worker_config* wc)
static void load_configuration_dynamic(void) static void load_configuration_dynamic(void)
{ {
unsigned read_num; unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
static int printed_error = 0; if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
read_num = (unsigned) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine extent pages",
MAX_PAGES_PER_EXTENT);
if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT) {
pages_per_extent = read_num; pages_per_extent = read_num;
} else if (!printed_error) { else {
printed_error = 1; error("Invalid dbengine pages per extent %u given. Using %u.", read_num, pages_per_extent);
error("Invalid dbengine extent pages %u given. Defaulting to %u.", read_num, pages_per_extent); config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", pages_per_extent);
} }
} }

5
database/engine/rrdengineapi.c
View File

@ -195,7 +195,7 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
handle->descr = NULL; handle->descr = NULL;
} }
void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, calculated_number n, SN_FLAGS flags) void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE n, SN_FLAGS flags)
{ {
storage_number number = pack_storage_number(n, flags); storage_number number = pack_storage_number(n, flags);
@ -603,7 +603,8 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
// Returns the metric and sets its timestamp into current_time // Returns the metric and sets its timestamp into current_time
// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags) // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
calculated_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) { NETDATA_DOUBLE
rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {
struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle; struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
struct rrdeng_page_descr *descr = handle->descr; struct rrdeng_page_descr *descr = handle->descr;

4
database/engine/rrdengineapi.h
View File

@ -42,14 +42,14 @@ extern void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN +
extern void rrdeng_metric_init(RRDDIM *rd); extern void rrdeng_metric_init(RRDDIM *rd);
extern void rrdeng_store_metric_init(RRDDIM *rd); extern void rrdeng_store_metric_init(RRDDIM *rd);
extern void rrdeng_store_metric_flush_current_page(RRDDIM *rd); extern void rrdeng_store_metric_flush_current_page(RRDDIM *rd);
extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags); extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
extern int rrdeng_store_metric_finalize(RRDDIM *rd); extern int rrdeng_store_metric_finalize(RRDDIM *rd);
extern unsigned extern unsigned
rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time, rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list); struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);
extern void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle, extern void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle,
time_t start_time, time_t end_time); time_t start_time, time_t end_time);
extern calculated_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags); extern NETDATA_DOUBLE rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
extern int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle); extern int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle);
extern void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle); extern void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
extern time_t rrdeng_metric_latest_time(RRDDIM *rd); extern time_t rrdeng_metric_latest_time(RRDDIM *rd);

66
database/metric_correlations.c
View File

@ -9,7 +9,7 @@ int metric_correlations_version = 1;
METRIC_CORRELATIONS_METHOD default_metric_correlations_method = METRIC_CORRELATIONS_KS2; METRIC_CORRELATIONS_METHOD default_metric_correlations_method = METRIC_CORRELATIONS_KS2;
typedef struct mc_stats { typedef struct mc_stats {
calculated_number max_base_high_ratio; NETDATA_DOUBLE max_base_high_ratio;
size_t db_points; size_t db_points;
size_t result_points; size_t result_points;
size_t db_queries; size_t db_queries;
@ -58,7 +58,7 @@ struct register_result {
const char *chart_id; const char *chart_id;
const char *context; const char *context;
const char *dim_name; const char *dim_name;
calculated_number value; NETDATA_DOUBLE value;
}; };
static void register_result_insert_callback(const char *name, void *value, void *data) { static void register_result_insert_callback(const char *name, void *value, void *data) {
@ -93,11 +93,11 @@ static void register_result_destroy(DICTIONARY *results) {
dictionary_destroy(results); dictionary_destroy(results);
} }
static void register_result(DICTIONARY *results, RRDSET *st, RRDDIM *d, calculated_number value, RESULT_FLAGS flags, MC_STATS *stats) { static void register_result(DICTIONARY *results, RRDSET *st, RRDDIM *d, NETDATA_DOUBLE value, RESULT_FLAGS flags, MC_STATS *stats) {
if(!calculated_number_isnumber(value)) return; if(!netdata_double_isnumber(value)) return;
// make it positive // make it positive
calculated_number v = calculated_number_fabs(value); NETDATA_DOUBLE v = fabsndd(value);
// no need to store zero scored values // no need to store zero scored values
if(v == 0.0) return; if(v == 0.0) return;
@ -186,7 +186,7 @@ static size_t registered_results_to_json(DICTIONARY *results, BUFFER *wb,
chart_dims = 0; chart_dims = 0;
} }
if (chart_dims) buffer_sprintf(wb, ",\n"); if (chart_dims) buffer_sprintf(wb, ",\n");
buffer_sprintf(wb, "\t\t\t\t\"%s\": " CALCULATED_NUMBER_FORMAT, t->dim_name, t->value); buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, t->dim_name, t->value);
chart_dims++; chart_dims++;
total_dimensions++; total_dimensions++;
} }
@ -240,14 +240,14 @@ int compare_diffs(const void *left, const void *right) {
return (lt > rt) - (lt < rt); return (lt > rt) - (lt < rt);
} }
static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, calculated_number *arr, size_t size) { static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, NETDATA_DOUBLE *arr, size_t size) {
calculated_number *last = &arr[size - 1]; NETDATA_DOUBLE *last = &arr[size - 1];
size_t added = 0; size_t added = 0;
while(last > arr) { while(last > arr) {
calculated_number second = *last--; NETDATA_DOUBLE second = *last--;
calculated_number first = *last; NETDATA_DOUBLE first = *last;
*diffs++ = (DIFFS_NUMBERS)((first - second) * (calculated_number)DOUBLE_TO_INT_MULTIPLIER); *diffs++ = (DIFFS_NUMBERS)((first - second) * (NETDATA_DOUBLE)DOUBLE_TO_INT_MULTIPLIER);
added++; added++;
} }
@ -356,7 +356,9 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB
return KSfbar((int)en, d); return KSfbar((int)en, d);
} }
static double kstwo(calculated_number baseline[], int baseline_points, calculated_number highlight[], int highlight_points, uint32_t base_shifts) { static double kstwo(
NETDATA_DOUBLE baseline[], int baseline_points,
NETDATA_DOUBLE highlight[], int highlight_points, uint32_t base_shifts) {
// -1 in size, since the calculate_pairs_diffs() returns one less point // -1 in size, since the calculate_pairs_diffs() returns one less point
DIFFS_NUMBERS baseline_diffs[baseline_points - 1]; DIFFS_NUMBERS baseline_diffs[baseline_points - 1];
DIFFS_NUMBERS highlight_diffs[highlight_points - 1]; DIFFS_NUMBERS highlight_diffs[highlight_points - 1];
@ -469,14 +471,14 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
// copy the baseline points of the dimension to a contiguous array // copy the baseline points of the dimension to a contiguous array
// there is no need to check for empty values, since empty are already zero // there is no need to check for empty values, since empty are already zero
calculated_number baseline[base_points]; NETDATA_DOUBLE baseline[base_points];
for(int c = 0; c < base_points; c++) for(int c = 0; c < base_points; c++)
baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ]; baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ];
// copy the highlight points of the dimension to a contiguous array // copy the highlight points of the dimension to a contiguous array
// there is no need to check for empty values, since empty values are already zero // there is no need to check for empty values, since empty values are already zero
// https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43 // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
calculated_number highlight[high_points]; NETDATA_DOUBLE highlight[high_points];
for(int c = 0; c < high_points; c++) for(int c = 0; c < high_points; c++)
highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ]; highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ];
@ -537,7 +539,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
// dimensions, and we query a single dimension at a time. // dimensions, and we query a single dimension at a time.
stats->db_queries++; stats->db_queries++;
calculated_number baseline_average = NAN; NETDATA_DOUBLE baseline_average = NAN;
uint8_t base_anomaly_rate = 0; uint8_t base_anomaly_rate = 0;
value_is_null = 1; value_is_null = 1;
ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1, ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1,
@ -547,13 +549,13 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
&stats->db_points, &stats->result_points, &stats->db_points, &stats->result_points,
&value_is_null, &base_anomaly_rate, 0); &value_is_null, &base_anomaly_rate, 0);
if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(baseline_average)) { if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(baseline_average)) {
// this means no data for the baseline window, but we may have data for the highlighted one - assume zero // this means no data for the baseline window, but we may have data for the highlighted one - assume zero
baseline_average = 0.0; baseline_average = 0.0;
} }
stats->db_queries++; stats->db_queries++;
calculated_number highlight_average = NAN; NETDATA_DOUBLE highlight_average = NAN;
uint8_t high_anomaly_rate = 0; uint8_t high_anomaly_rate = 0;
value_is_null = 1; value_is_null = 1;
ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1, ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1,
@ -563,7 +565,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
&stats->db_points, &stats->result_points, &stats->db_points, &stats->result_points,
&value_is_null, &high_anomaly_rate, 0); &value_is_null, &high_anomaly_rate, 0);
if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(highlight_average)) { if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_average)) {
// this means no data for the highlighted duration - so skip it // this means no data for the highlighted duration - so skip it
continue; continue;
} }
@ -574,11 +576,11 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
} }
stats->db_queries++; stats->db_queries++;
calculated_number highlight_countif = NAN; NETDATA_DOUBLE highlight_countif = NAN;
value_is_null = 1; value_is_null = 1;
char highlighted_countif_options[50 + 1]; char highlighted_countif_options[50 + 1];
snprintfz(highlighted_countif_options, 50, "%s" CALCULATED_NUMBER_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average); snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average);
ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1, ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1,
after, before, after, before,
@ -588,7 +590,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
&stats->db_points, &stats->result_points, &stats->db_points, &stats->result_points,
&value_is_null, NULL, 0); &value_is_null, NULL, 0);
if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(highlight_countif)) { if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_countif)) {
info("MC: highlighted countif query failed, but highlighted average worked - strange..."); info("MC: highlighted countif query failed, but highlighted average worked - strange...");
continue; continue;
} }
@ -599,7 +601,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0 highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0
RESULT_FLAGS flags; RESULT_FLAGS flags;
calculated_number pcent = NAN; NETDATA_DOUBLE pcent = NAN;
if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) { if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) {
flags = RESULT_IS_BASE_HIGH_RATIO; flags = RESULT_IS_BASE_HIGH_RATIO;
pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif; pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif;
@ -615,15 +617,15 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
return correlated_dimensions; return correlated_dimensions;
} }
int compare_calculated_numbers(const void *left, const void *right) { int compare_netdata_doubles(const void *left, const void *right) {
calculated_number lt = *(calculated_number *)left; NETDATA_DOUBLE lt = *(NETDATA_DOUBLE *)left;
calculated_number rt = *(calculated_number *)right; NETDATA_DOUBLE rt = *(NETDATA_DOUBLE *)right;
// https://stackoverflow.com/a/3886497/1114110 // https://stackoverflow.com/a/3886497/1114110
return (lt > rt) - (lt < rt); return (lt > rt) - (lt < rt);
} }
static inline int binary_search_bigger_than_calculated_number(const calculated_number arr[], int left, int size, calculated_number K) { static inline int binary_search_bigger_than_netdata_double(const NETDATA_DOUBLE arr[], int left, int size, NETDATA_DOUBLE K) {
// binary search to find the index the smallest index // binary search to find the index the smallest index
// of the first value in the array that is greater than K // of the first value in the array that is greater than K
@ -655,7 +657,7 @@ static size_t spread_results_evenly(DICTIONARY *results, MC_STATS *stats) {
stats->max_base_high_ratio = 1.0; stats->max_base_high_ratio = 1.0;
// create an array of the right size and copy all the values in it // create an array of the right size and copy all the values in it
calculated_number slots[dimensions]; NETDATA_DOUBLE slots[dimensions];
dimensions = 0; dimensions = 0;
dfe_start_read(results, t) { dfe_start_read(results, t) {
if(t->flags & (RESULT_IS_PERCENTAGE_OF_TIME)) if(t->flags & (RESULT_IS_PERCENTAGE_OF_TIME))
@ -666,10 +668,10 @@ static size_t spread_results_evenly(DICTIONARY *results, MC_STATS *stats) {
dfe_done(t); dfe_done(t);
// sort the array with the values of all dimensions // sort the array with the values of all dimensions
qsort(slots, dimensions, sizeof(calculated_number), compare_calculated_numbers); qsort(slots, dimensions, sizeof(NETDATA_DOUBLE), compare_netdata_doubles);
// skip the duplicates in the sorted array // skip the duplicates in the sorted array
calculated_number last_value = NAN; NETDATA_DOUBLE last_value = NAN;
size_t unique_values = 0; size_t unique_values = 0;
for(size_t i = 0; i < dimensions ;i++) { for(size_t i = 0; i < dimensions ;i++) {
if(likely(slots[i] != last_value)) if(likely(slots[i] != last_value))
@ -681,11 +683,11 @@ static size_t spread_results_evenly(DICTIONARY *results, MC_STATS *stats) {
unique_values = dimensions; unique_values = dimensions;
// calculate the weight of each slot, using the number of unique values // calculate the weight of each slot, using the number of unique values
calculated_number slot_weight = 1.0 / (calculated_number)unique_values; NETDATA_DOUBLE slot_weight = 1.0 / (NETDATA_DOUBLE)unique_values;
dfe_start_read(results, t) { dfe_start_read(results, t) {
int slot = binary_search_bigger_than_calculated_number(slots, 0, (int)unique_values, t->value); int slot = binary_search_bigger_than_netdata_double(slots, 0, (int)unique_values, t->value);
calculated_number v = slot * slot_weight; NETDATA_DOUBLE v = slot * slot_weight;
if(unlikely(v > 1.0)) v = 1.0; if(unlikely(v > 1.0)) v = 1.0;
v = 1.0 - v; v = 1.0 - v;
t->value = v; t->value = v;

11
database/ram/rrddim_mem.c
View File

@ -6,12 +6,12 @@
// RRDDIM legacy data collection functions // RRDDIM legacy data collection functions
void rrddim_collect_init(RRDDIM *rd) { void rrddim_collect_init(RRDDIM *rd) {
rd->values[rd->rrdset->current_entry] = SN_EMPTY_SLOT; rd->db[rd->rrdset->current_entry] = SN_EMPTY_SLOT;
rd->state->handle = calloc(1, sizeof(struct mem_collect_handle)); rd->state->handle = calloc(1, sizeof(struct mem_collect_handle));
} }
void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags) { void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags) {
(void)point_in_time; (void)point_in_time;
rd->values[rd->rrdset->current_entry] = pack_storage_number(number, flags); rd->db[rd->rrdset->current_entry] = pack_storage_number(number, flags);
} }
int rrddim_collect_finalize(RRDDIM *rd) { int rrddim_collect_finalize(RRDDIM *rd) {
free((struct mem_collect_handle*)rd->state->handle); free((struct mem_collect_handle*)rd->state->handle);
@ -42,7 +42,8 @@ void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t st
// Returns the metric and sets its timestamp into current_time // Returns the metric and sets its timestamp into current_time
// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags) // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
calculated_number rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) { NETDATA_DOUBLE
rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {
RRDDIM *rd = handle->rd; RRDDIM *rd = handle->rd;
struct mem_query_handle* h = (struct mem_query_handle*)handle->handle; struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;
size_t entries = rd->rrdset->entries; size_t entries = rd->rrdset->entries;
@ -65,7 +66,7 @@ calculated_number rrddim_query_next_metric(struct rrddim_query_handle *handle, t
return NAN; return NAN;
} }
storage_number n = rd->values[slot++]; storage_number n = rd->db[slot++];
if(unlikely(slot >= entries)) slot = 0; if(unlikely(slot >= entries)) slot = 0;
h->slot = slot; h->slot = slot;

5
database/ram/rrddim_mem.h
View File

@ -20,11 +20,12 @@ struct mem_query_handle {
}; };
extern void rrddim_collect_init(RRDDIM *rd); extern void rrddim_collect_init(RRDDIM *rd);
extern void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags); extern void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
extern int rrddim_collect_finalize(RRDDIM *rd); extern int rrddim_collect_finalize(RRDDIM *rd);
extern void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time); extern void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
extern calculated_number rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags); extern NETDATA_DOUBLE
rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
extern int rrddim_query_is_finished(struct rrddim_query_handle *handle); extern int rrddim_query_is_finished(struct rrddim_query_handle *handle);
extern void rrddim_query_finalize(struct rrddim_query_handle *handle); extern void rrddim_query_finalize(struct rrddim_query_handle *handle);
extern time_t rrddim_query_latest_time(RRDDIM *rd); extern time_t rrddim_query_latest_time(RRDDIM *rd);

94
database/rrd.h
View File

@ -74,9 +74,6 @@ extern time_t rrdset_free_obsolete_time;
#define RRD_ID_LENGTH_MAX 200 #define RRD_ID_LENGTH_MAX 200
#define RRDSET_MAGIC "NETDATA RRD SET FILE V019"
#define RRDDIMENSION_MAGIC "NETDATA RRD DIMENSION FILE V019"
typedef long long total_number; typedef long long total_number;
#define TOTAL_NUMBER_FORMAT "%lld" #define TOTAL_NUMBER_FORMAT "%lld"
@ -237,52 +234,44 @@ struct rrddim {
// this is a pointer to the config structure // this is a pointer to the config structure
// since the config always has a higher priority // since the config always has a higher priority
// (the user overwrites the name of the charts) // (the user overwrites the name of the charts)
// DO NOT FREE THIS - IT IS ALLOCATED IN CONFIG uint32_t hash; // a simple hash of the id, to speed up searching / indexing
// instead of strcmp() every item in the binary index
// we first compare the hashes
uint32_t hash_name; // a simple hash of the name
RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
RRDDIM_FLAGS flags; // configuration flags for the dimension
unsigned int updated:1; // 1 when the dimension has been updated since the last processing
unsigned int exposed:1; // 1 when set what have sent this dimension to the central netdata
collected_number multiplier; // the multiplier of the collected values collected_number multiplier; // the multiplier of the collected values
collected_number divisor; // the divider of the collected values collected_number divisor; // the divider of the collected values
uint32_t flags; // configuration flags for the dimension
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// members for temporary data we need for calculations // members for temporary data we need for calculations
uint32_t hash; // a simple hash of the id, to speed up searching / indexing
// instead of strcmp() every item in the binary index
// we first compare the hashes
uint32_t hash_name; // a simple hash of the name
char *cache_filename; // the filename we load/save from/to this set
size_t collections_counter; // the number of times we added values to this rrdim
struct rrddim_volatile *state; // volatile state that is not persistently stored
size_t unused[8];
collected_number collected_value_max; // the absolute maximum of the collected value
unsigned int updated:1; // 1 when the dimension has been updated since the last processing
unsigned int exposed:1; // 1 when set what have sent this dimension to the central netdata
struct timeval last_collected_time; // when was this dimension last updated struct timeval last_collected_time; // when was this dimension last updated
// this is actual date time we updated the last_collected_value // this is actual date time we updated the last_collected_value
// THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
calculated_number calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used struct rrddim_volatile *state; // volatile state that is not persistently stored
calculated_number last_calculated_value; // the last calculated value processed size_t collections_counter; // the number of times we added values to this rrdim
collected_number collected_value_max; // the absolute maximum of the collected value
calculated_number last_stored_value; // the last value as stored in the database (after interpolation) NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
NETDATA_DOUBLE last_calculated_value; // the last calculated value processed
NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation)
collected_number collected_value; // the current value, as collected - resets to 0 after being used collected_number collected_value; // the current value, as collected - resets to 0 after being used
collected_number last_collected_value; // the last value that was collected, after being processed collected_number last_collected_value; // the last value that was collected, after being processed
// the *_volume members are used to calculate the accuracy of the rounding done by the // the *_volume members are used to calculate the accuracy of the rounding done by the
// storage number - they are printed to debug.log when debug is enabled for a set. // storage number - they are printed to debug.log when debug is enabled for a set.
calculated_number collected_volume; // the sum of all collected values so far NETDATA_DOUBLE collected_volume; // the sum of all collected values so far
calculated_number stored_volume; // the sum of all stored values so far NETDATA_DOUBLE stored_volume; // the sum of all stored values so far
struct rrddim *next; // linking of dimensions within the same data set struct rrddim *next; // linking of dimensions within the same data set
struct rrdset *rrdset; struct rrdset *rrdset;
@ -296,18 +285,33 @@ struct rrddim {
int update_every; // every how many seconds is this updated int update_every; // every how many seconds is this updated
size_t memsize; // the memory allocated for this dimension size_t memsize; // the memory allocated for this dimension (without RRDDIM)
char magic[sizeof(RRDDIMENSION_MAGIC) + 1]; // a string to be saved, used to identify our data file
struct rrddimvar *variables; struct rrddimvar *variables;
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// the values stored in this dimension, using our floating point numbers // the values stored in this dimension, using our floating point numbers
storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER void *rd_on_file; // pointer to the header written on disk
storage_number *db; // the array of values
}; };
// returns the RRDDIM cache filename, or NULL if it does not exist
extern const char *rrddim_cache_filename(RRDDIM *rd);
// updated the header with the latest RRDDIM value, for memory mode MAP and SAVE
extern void rrddim_memory_file_update(RRDDIM *rd);
// free the memory file structures for memory mode MAP and SAVE
extern void rrddim_memory_file_free(RRDDIM *rd);
extern bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode);
// return the v019 header size of RRDDIM files
extern size_t rrddim_memory_file_header_size(void);
extern void rrddim_memory_file_save(RRDDIM *rd);
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// engine-specific iterator state for dimension data collection // engine-specific iterator state for dimension data collection
typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE; typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
@ -332,7 +336,7 @@ struct rrddim_collect_ops {
void (*init)(RRDDIM *rd); void (*init)(RRDDIM *rd);
// run this to store each metric into the database // run this to store each metric into the database
void (*store_metric)(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags); void (*store_metric)(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
// an finalization function to run after collection is over // an finalization function to run after collection is over
// returns 1 if it's safe to delete the dimension // returns 1 if it's safe to delete the dimension
@ -345,7 +349,7 @@ struct rrddim_query_ops {
void (*init)(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time); void (*init)(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
// run this to load each metric number from the database // run this to load each metric number from the database
calculated_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time, time_t *end_time, SN_FLAGS *flags); NETDATA_DOUBLE (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time, time_t *end_time, SN_FLAGS *flags);
// run this to test if the series of next_metric() database queries is finished // run this to test if the series of next_metric() database queries is finished
int (*is_finished)(struct rrddim_query_handle *handle); int (*is_finished)(struct rrddim_query_handle *handle);
@ -450,8 +454,6 @@ struct rrdset {
// since the config always has a higher priority // since the config always has a higher priority
// (the user overwrites the name of the charts) // (the user overwrites the name of the charts)
void *unused_ptr; // Unused field (previously it held the config section of the chart)
char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options)
char *family; // grouping sets under the same family char *family; // grouping sets under the same family
char *title; // title shown to user char *title; // title shown to user
@ -484,7 +486,6 @@ struct rrdset {
RRD_MEMORY_MODE rrd_memory_mode; // if set to 1, this is memory mapped RRD_MEMORY_MODE rrd_memory_mode; // if set to 1, this is memory mapped
char *cache_dir; // the directory to store dimensions char *cache_dir; // the directory to store dimensions
char cache_filename[FILENAME_MAX+1]; // the filename to store this set
netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list
@ -502,7 +503,6 @@ struct rrdset {
uuid_t *chart_uuid; // Store the global GUID for this chart uuid_t *chart_uuid; // Store the global GUID for this chart
// this object. // this object.
struct rrdset_volatile *state; // volatile state that is not persistently stored struct rrdset_volatile *state; // volatile state that is not persistently stored
size_t unused[3];
size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine
@ -527,8 +527,8 @@ struct rrdset {
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// local variables // local variables
calculated_number green; // green threshold for this chart NETDATA_DOUBLE green; // green threshold for this chart
calculated_number red; // red threshold for this chart NETDATA_DOUBLE red; // red threshold for this chart
avl_tree_lock rrdvar_root_index; // RRDVAR index for this chart avl_tree_lock rrdvar_root_index; // RRDVAR index for this chart
RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs) RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs)
@ -538,15 +538,13 @@ struct rrdset {
// members for checking the data when loading from disk // members for checking the data when loading from disk
unsigned long memsize; // how much mem we have allocated for this (without dimensions) unsigned long memsize; // how much mem we have allocated for this (without dimensions)
void *st_on_file; // compatibility with V019 RRDSET files
char magic[sizeof(RRDSET_MAGIC) + 1]; // our magic
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// the dimensions // the dimensions
avl_tree_lock dimensions_index; // the root of the dimensions index avl_tree_lock dimensions_index; // the root of the dimensions index
RRDDIM *dimensions; // the actual data for every dimension RRDDIM *dimensions; // the actual data for every dimension
}; };
#define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock)) #define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock))
@ -564,6 +562,12 @@ struct rrdset {
for((st) = (host)->rrdset_root, rrdhost_check_wrlock(host); st ; (st) = (st)->next) for((st) = (host)->rrdset_root, rrdhost_check_wrlock(host); st ; (st) = (st)->next)
extern void rrdset_memory_file_save(RRDSET *st);
extern void rrdset_memory_file_free(RRDSET *st);
extern void rrdset_memory_file_update(RRDSET *st);
extern const char *rrdset_cache_filename(RRDSET *st);
extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE memory_mode);
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// RRDHOST flags // RRDHOST flags
// use this for configuration flags, not for state control // use this for configuration flags, not for state control
@ -630,8 +634,8 @@ struct alarm_entry {
char *units; char *units;
char *info; char *info;
calculated_number old_value; NETDATA_DOUBLE old_value;
calculated_number new_value; NETDATA_DOUBLE new_value;
char *old_value_string; char *old_value_string;
char *new_value_string; char *new_value_string;

10
database/rrdcalc.c
View File

@ -57,12 +57,14 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
} }
if(!isnan(rc->green) && isnan(st->green)) { if(!isnan(rc->green) && isnan(st->green)) {
debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " CALCULATED_NUMBER_FORMAT_AUTO " to " CALCULATED_NUMBER_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green); debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " NETDATA_DOUBLE_FORMAT_AUTO
" to " NETDATA_DOUBLE_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green);
st->green = rc->green; st->green = rc->green;
} }
if(!isnan(rc->red) && isnan(st->red)) { if(!isnan(rc->red) && isnan(st->red)) {
debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " CALCULATED_NUMBER_FORMAT_AUTO " to " CALCULATED_NUMBER_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red); debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " NETDATA_DOUBLE_FORMAT_AUTO " to " NETDATA_DOUBLE_FORMAT_AUTO
".", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red);
st->red = rc->red; st->red = rc->red;
} }
@ -445,7 +447,9 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt,
error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source); error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source);
} }
debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO
", red " NETDATA_DOUBLE_FORMAT_AUTO
", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
(rc->chart)?rc->chart:"NOCHART", (rc->chart)?rc->chart:"NOCHART",
rc->name, rc->name,
(rc->exec)?rc->exec:"DEFAULT", (rc->exec)?rc->exec:"DEFAULT",

8
database/rrdcalc.h
View File

@ -63,8 +63,8 @@ struct rrdcalc {
int update_every; // update frequency for the alarm int update_every; // update frequency for the alarm
// the red and green threshold of this alarm (to be set to the chart) // the red and green threshold of this alarm (to be set to the chart)
calculated_number green; NETDATA_DOUBLE green;
calculated_number red; NETDATA_DOUBLE red;
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// database lookup settings // database lookup settings
@ -112,8 +112,8 @@ struct rrdcalc {
RRDCALC_STATUS old_status; // the old status of the alarm RRDCALC_STATUS old_status; // the old status of the alarm
RRDCALC_STATUS status; // the current status of the alarm RRDCALC_STATUS status; // the current status of the alarm
calculated_number value; // the current value of the alarm NETDATA_DOUBLE value; // the current value of the alarm
calculated_number old_value; // the previous value of the alarm NETDATA_DOUBLE old_value; // the previous value of the alarm
uint32_t rrdcalc_flags; // check RRDCALC_FLAG_* uint32_t rrdcalc_flags; // check RRDCALC_FLAG_*

4
database/rrdcalctemplate.h
View File

@ -42,8 +42,8 @@ struct rrdcalctemplate {
int update_every; // update frequency for the alarm int update_every; // update frequency for the alarm
// the red and green threshold of this alarm (to be set to the chart) // the red and green threshold of this alarm (to be set to the chart)
calculated_number green; NETDATA_DOUBLE green;
calculated_number red; NETDATA_DOUBLE red;
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// database lookup settings // database lookup settings

349
database/rrddim.c
View File

@ -197,112 +197,15 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
char filename[FILENAME_MAX + 1]; rd = callocz(1, sizeof(RRDDIM));
char fullfilename[FILENAME_MAX + 1];
unsigned long size = sizeof(RRDDIM) + (st->entries * sizeof(storage_number));
debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id);
rrdset_strncpyz_name(filename, id, FILENAME_MAX);
snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP ||
memory_mode == RRD_MEMORY_MODE_RAM) {
rd = (RRDDIM *)netdata_mmap(
(memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename,
size,
((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
1);
if(likely(rd)) {
// we have a file mapped for rd
memset(&rd->avl, 0, sizeof(avl_t));
rd->id = NULL;
rd->name = NULL;
rd->cache_filename = NULL;
rd->variables = NULL;
rd->next = NULL;
rd->rrdset = NULL;
rd->exposed = 0;
struct timeval now;
now_realtime_timeval(&now);
if(memory_mode == RRD_MEMORY_MODE_RAM) {
memset(rd, 0, size);
}
else {
int reset = 0;
if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) {
info("Initializing file %s.", fullfilename);
memset(rd, 0, size);
reset = 1;
}
else if(rd->memsize != size) {
error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, rd->memsize);
memset(rd, 0, size);
reset = 1;
}
else if(rd->update_every != st->update_every) {
error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd->update_every);
memset(rd, 0, size);
reset = 1;
}
else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) {
info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd->last_collected_time) / USEC_PER_SEC, rd->entries * rd->update_every);
memset(rd, 0, size);
reset = 1;
}
if(!reset) {
if(rd->algorithm != algorithm) {
info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
fullfilename, algorithm, rrd_algorithm_name(algorithm), rd->algorithm, rrd_algorithm_name(rd->algorithm));
}
if(rd->multiplier != multiplier) {
info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, multiplier, rd->multiplier);
}
if(rd->divisor != divisor) {
info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, divisor, rd->divisor);
}
}
}
// make sure we have the right memory mode
// even if we cleared the memory
rd->rrd_memory_mode = memory_mode;
}
}
if(unlikely(!rd)) {
// if we didn't manage to get a mmap'd dimension, just create one
rd = callocz(1, size);
if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
else
rd->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
rd->memsize = size;
strcpy(rd->magic, RRDDIMENSION_MAGIC);
rd->id = strdupz(id); rd->id = strdupz(id);
rd->hash = simple_hash(rd->id); rd->hash = simple_hash(rd->id);
rd->cache_filename = strdupz(fullfilename);
rd->name = (name && *name)?strdupz(name):strdupz(rd->id); rd->name = (name && *name)?strdupz(name):strdupz(rd->id);
rd->hash_name = simple_hash(rd->name); rd->hash_name = simple_hash(rd->name);
rd->algorithm = algorithm; rd->algorithm = algorithm;
rd->multiplier = multiplier; rd->multiplier = multiplier;
rd->divisor = divisor; rd->divisor = divisor;
if(!rd->divisor) rd->divisor = 1; if(!rd->divisor) rd->divisor = 1;
@ -311,23 +214,38 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)) if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
rd->collections_counter = 1; rd->collections_counter = 1;
else
rd->collections_counter = 0;
rd->updated = 0;
rd->flags = 0x00000000;
rd->calculated_value = 0;
rd->last_calculated_value = 0;
rd->collected_value = 0;
rd->last_collected_value = 0;
rd->collected_value_max = 0;
rd->collected_volume = 0;
rd->stored_volume = 0;
rd->last_stored_value = 0;
rd->last_collected_time.tv_sec = 0;
rd->last_collected_time.tv_usec = 0;
rd->rrdset = st; rd->rrdset = st;
if(memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_SAVE) {
if(!rrddim_memory_load_or_create_map_save(st, rd, memory_mode)) {
info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name, rd->name);
memory_mode = RRD_MEMORY_MODE_RAM;
}
}
if(memory_mode == RRD_MEMORY_MODE_RAM) {
size_t entries = st->entries;
if(!entries) entries = 5;
rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1);
if(!rd->db) {
info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", st->name, rd->name);
memory_mode = RRD_MEMORY_MODE_ALLOC;
}
else rd->memsize = entries * sizeof(storage_number);
}
if(memory_mode == RRD_MEMORY_MODE_ALLOC || memory_mode == RRD_MEMORY_MODE_NONE) {
size_t entries = st->entries;
if(entries < 5) entries = 5;
rd->db = callocz(entries, sizeof(storage_number));
rd->memsize = entries * sizeof(storage_number);
}
rd->rrd_memory_mode = memory_mode;
rd->state = callocz(1, sizeof(*rd->state)); rd->state = callocz(1, sizeof(*rd->state));
#ifdef ENABLE_ACLK #ifdef ENABLE_ACLK
rd->state->aclk_live_status = -1; rd->state->aclk_live_status = -1;
@ -432,30 +350,21 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
// aclk_send_dimension_update(rd); // aclk_send_dimension_update(rd);
//#endif //#endif
RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode; freez((void *)rd->id);
switch(rrd_memory_mode) { freez((void *)rd->name);
case RRD_MEMORY_MODE_SAVE: freez(rd->state);
case RRD_MEMORY_MODE_MAP:
case RRD_MEMORY_MODE_RAM:
debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
freez((void *)rd->id);
freez((void *)rd->name);
freez(rd->cache_filename);
freez(rd->state);
munmap(rd, rd->memsize);
break;
case RRD_MEMORY_MODE_ALLOC: // this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures
case RRD_MEMORY_MODE_NONE: rrddim_memory_file_free(rd);
case RRD_MEMORY_MODE_DBENGINE:
debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name); if(rd->db) {
freez((void *)rd->id); if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
freez((void *)rd->name); munmap(rd->db, rd->memsize);
freez(rd->cache_filename); else
freez(rd->state); freez(rd->db);
freez(rd);
break;
} }
freez(rd);
} }
@ -529,7 +438,7 @@ inline collected_number rrddim_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM
collected_number v = (value >= 0) ? value : -value; collected_number v = (value >= 0) ? value : -value;
if(unlikely(v > rd->collected_value_max)) rd->collected_value_max = v; if(unlikely(v > rd->collected_value_max)) rd->collected_value_max = v;
// fprintf(stderr, "%s.%s %llu " COLLECTED_NUMBER_FORMAT " dt %0.6f" " rate " CALCULATED_NUMBER_FORMAT "\n", st->name, rd->name, st->usec_since_last_update, value, (float)((double)st->usec_since_last_update / (double)1000000), (calculated_number)((value - rd->last_collected_value) * (calculated_number)rd->multiplier / (calculated_number)rd->divisor * 1000000.0 / (calculated_number)st->usec_since_last_update)); // fprintf(stderr, "%s.%s %llu " COLLECTED_NUMBER_FORMAT " dt %0.6f" " rate " NETDATA_DOUBLE_FORMAT "\n", st->name, rd->name, st->usec_since_last_update, value, (float)((double)st->usec_since_last_update / (double)1000000), (NETDATA_DOUBLE)((value - rd->last_collected_value) * (NETDATA_DOUBLE)rd->multiplier / (NETDATA_DOUBLE)rd->divisor * 1000000.0 / (NETDATA_DOUBLE)st->usec_since_last_update));
return rd->last_collected_value; return rd->last_collected_value;
} }
@ -544,3 +453,171 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value)
return rrddim_set_by_pointer(st, rd, value); return rrddim_set_by_pointer(st, rd, value);
} }
// ----------------------------------------------------------------------------
// compatibility layer for RRDDIM files v019
#define RRDDIMENSION_MAGIC_V019 "NETDATA RRD DIMENSION FILE V019"
struct avl_element_v019 {
void *avl_link[2];
signed char avl_balance;
};
struct rrddim_map_save_v019 {
struct avl_element_v019 avl; // ignored
void *id; // ignored
void *name; // ignored
uint32_t algorithm; // print warning on mismatch - update on load
uint32_t rrd_memory_mode; // ignored
long long multiplier; // print warning on mismatch - update on load
long long divisor; // print warning on mismatch - update on load
uint32_t flags; // ignored
uint32_t hash; // ignored
uint32_t hash_name; // ignored
void *cache_filename; // ignored - we use it to keep the filename to save back
size_t collections_counter; // ignored
void *state; // ignored
size_t unused[8]; // ignored
long long collected_value_max; // ignored
unsigned int updated:1; // ignored
unsigned int exposed:1; // ignored
struct timeval last_collected_time; // check to reset all - ignored after load
long double calculated_value; // ignored
long double last_calculated_value; // ignored
long double last_stored_value; // ignored
long long collected_value; // ignored
long long last_collected_value; // ignored
long double collected_volume; // ignored
long double stored_volume; // ignored
void *next; // ignored
void *rrdset; // ignored
long entries; // check to reset all - update on load
int update_every; // check to reset all - update on load
size_t memsize; // check to reset all - update on load
char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load
void *variables; // ignored
storage_number values[]; // the array of values
};
size_t rrddim_memory_file_header_size(void) {
return sizeof(struct rrddim_map_save_v019);
}
void rrddim_memory_file_update(RRDDIM *rd) {
if(!rd->rd_on_file) return;
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec;
rd_on_file->last_collected_time.tv_usec = rd->last_collected_time.tv_usec;
}
void rrddim_memory_file_free(RRDDIM *rd) {
if(!rd->rd_on_file) return;
// needed for memory mode map, to save the latest state
rrddim_memory_file_update(rd);
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
freez(rd_on_file->cache_filename);
munmap(rd_on_file, rd_on_file->memsize);
// remove the pointers from the RRDDIM
rd->rd_on_file = NULL;
rd->db = NULL;
}
const char *rrddim_cache_filename(RRDDIM *rd) {
if(!rd->rd_on_file) return NULL;
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
return rd_on_file->cache_filename;
}
void rrddim_memory_file_save(RRDDIM *rd) {
if(!rd->rd_on_file) return;
rrddim_memory_file_update(rd);
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
if(rd_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize);
}
bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode) {
if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
return false;
struct rrddim_map_save_v019 *rd_on_file = NULL;
unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->entries * sizeof(storage_number));
char filename[FILENAME_MAX + 1];
char fullfilename[FILENAME_MAX + 1];
rrdset_strncpyz_name(filename, rd->id, FILENAME_MAX);
snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(fullfilename, size,
((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1);
if(unlikely(!rd_on_file)) return false;
struct timeval now;
now_realtime_timeval(&now);
int reset = 0;
if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) {
info("Initializing file %s.", fullfilename);
memset(rd_on_file, 0, size);
reset = 1;
}
else if(rd_on_file->memsize != size) {
error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, rd_on_file->memsize);
memset(rd_on_file, 0, size);
reset = 1;
}
else if(rd_on_file->update_every != st->update_every) {
error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every);
memset(rd_on_file, 0, size);
reset = 1;
}
else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) {
info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every);
memset(rd_on_file, 0, size);
reset = 1;
}
if(!reset) {
if(rd_on_file->algorithm != rd->algorithm)
info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
if(rd_on_file->multiplier != rd->multiplier)
info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, rd->multiplier, rd_on_file->multiplier);
if(rd_on_file->divisor != rd->divisor)
info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, rd->divisor, rd_on_file->divisor);
}
// zero the entire header
memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019));
// set the important fields
strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019);
rd_on_file->algorithm = rd->algorithm;
rd_on_file->multiplier = rd->multiplier;
rd_on_file->divisor = rd->divisor;
rd_on_file->entries = st->entries;
rd_on_file->update_every = rd->update_every;
rd_on_file->memsize = size;
rd_on_file->rrd_memory_mode = memory_mode;
rd_on_file->cache_filename = strdupz(fullfilename);
rd->db = &rd_on_file->values[0];
rd->rd_on_file = rd_on_file;
rd->memsize = size;
rrddim_memory_file_update(rd);
return true;
}

16
database/rrdhost.c
View File

@ -210,10 +210,10 @@ RRDHOST *rrdhost_create(const char *hostname,
avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare); avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare);
avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare); avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare);
if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1)) if(config_get_boolean(CONFIG_SECTION_DB, "delete obsolete charts files", 1))
rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS);
if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", 1) && !is_localhost) if(config_get_boolean(CONFIG_SECTION_DB, "delete orphan hosts files", 1) && !is_localhost)
rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST); rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST);
host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never"); host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
@ -689,17 +689,21 @@ restart_after_removal:
// RRDHOST global / startup initialization // RRDHOST global / startup initialization
int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", rrdset_free_obsolete_time); rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
// Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short
// cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
// https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information. // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.
if (rrdset_free_obsolete_time < 10) { if (rrdset_free_obsolete_time < 10) {
rrdset_free_obsolete_time = 10; rrdset_free_obsolete_time = 10;
info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds. A lower delay can potentially cause a segmentation fault."); info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds.");
config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
} }
gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_GLOBAL, "gap when lost iterations above", gap_when_lost_iterations_above);
if (gap_when_lost_iterations_above < 1) gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
if (gap_when_lost_iterations_above < 1) {
gap_when_lost_iterations_above = 1; gap_when_lost_iterations_above = 1;
config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
}
if (unlikely(sql_init_database(DB_CHECK_NONE, 0))) { if (unlikely(sql_init_database(DB_CHECK_NONE, 0))) {
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

616
database/rrdset.c
View File

@ -80,17 +80,9 @@ static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name, ui
tmp.name = name; tmp.name = name;
tmp.hash_name = (hash)?hash:simple_hash(tmp.name); tmp.hash_name = (hash)?hash:simple_hash(tmp.name);
// fprintf(stderr, "SEARCHING: %s\n", name);
result = avl_search_lock(&host->rrdset_root_index_name, (avl_t *) (&(tmp.avlname))); result = avl_search_lock(&host->rrdset_root_index_name, (avl_t *) (&(tmp.avlname)));
if(result) { if(result) return rrdset_from_avlname(result);
RRDSET *st = rrdset_from_avlname(result);
if(strcmp(st->magic, RRDSET_MAGIC) != 0)
error("Search for RRDSET %s returned an invalid RRDSET %s (name %s)", name, st->id, st->name);
// fprintf(stderr, "FOUND: %s\n", name);
return rrdset_from_avlname(result);
}
// fprintf(stderr, "NOT FOUND: %s\n", name);
return NULL; return NULL;
} }
@ -294,20 +286,27 @@ void rrdset_reset(RRDSET *st) {
// RRDSET - helpers for rrdset_create() // RRDSET - helpers for rrdset_create()
inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) { inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
if(unlikely(entries < 5)) entries = 5; if(mode == RRD_MEMORY_MODE_DBENGINE) return 0;
if(unlikely(entries > RRD_HISTORY_ENTRIES_MAX)) entries = RRD_HISTORY_ENTRIES_MAX; if(mode == RRD_MEMORY_MODE_NONE) return 5;
if(unlikely(mode == RRD_MEMORY_MODE_NONE || mode == RRD_MEMORY_MODE_ALLOC)) if(entries < 5) entries = 5;
return entries; if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
long page = (size_t)sysconf(_SC_PAGESIZE); if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE || mode == RRD_MEMORY_MODE_RAM) {
long size = sizeof(RRDDIM) + entries * sizeof(storage_number); long header_size = 0;
if(unlikely(size % page)) {
size -= (size % page);
size += page;
long n = (size - sizeof(RRDDIM)) / sizeof(storage_number); if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE)
return n; header_size = (long)rrddim_memory_file_header_size();
long page = (long)sysconf(_SC_PAGESIZE);
long size = (long)(header_size + entries * sizeof(storage_number));
if (unlikely(size % page)) {
size -= (size % page);
size += page;
long n = (long)((size - header_size) / sizeof(storage_number));
return n;
}
} }
return entries; return entries;
@ -405,40 +404,18 @@ void rrdset_free(RRDSET *st) {
freez(st->state); freez(st->state);
freez(st->chart_uuid); freez(st->chart_uuid);
switch(st->rrd_memory_mode) { rrdset_memory_file_free(st);
case RRD_MEMORY_MODE_SAVE: freez(st);
case RRD_MEMORY_MODE_MAP:
case RRD_MEMORY_MODE_RAM:
debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
munmap(st, st->memsize);
break;
case RRD_MEMORY_MODE_ALLOC:
case RRD_MEMORY_MODE_NONE:
case RRD_MEMORY_MODE_DBENGINE:
freez(st);
break;
}
} }
void rrdset_save(RRDSET *st) { void rrdset_save(RRDSET *st) {
rrdset_check_rdlock(st); rrdset_check_rdlock(st);
// info("Saving chart '%s' ('%s')", st->id, st->name); rrdset_memory_file_save(st);
if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) {
debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename);
memory_file_save(st->cache_filename, st, st->memsize);
}
RRDDIM *rd; RRDDIM *rd;
rrddim_foreach_read(rd, st) { rrddim_foreach_read(rd, st)
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) { rrddim_memory_file_save(rd);
debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename);
memory_file_save(rd->cache_filename, rd, rd->memsize);
}
}
} }
void rrdset_delete(RRDSET *st) { void rrdset_delete(RRDSET *st) {
@ -448,17 +425,23 @@ void rrdset_delete(RRDSET *st) {
info("Deleting chart '%s' ('%s') from disk...", st->id, st->name); info("Deleting chart '%s' ('%s') from disk...", st->id, st->name);
if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) { if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
info("Deleting chart header file '%s'.", st->cache_filename); const char *cache_filename = rrdset_cache_filename(st);
if(unlikely(unlink(st->cache_filename) == -1)) if(cache_filename) {
error("Cannot delete chart header file '%s'", st->cache_filename); info("Deleting chart header file '%s'.", cache_filename);
if (unlikely(unlink(cache_filename) == -1))
error("Cannot delete chart header file '%s'", cache_filename);
}
else
error("Cannot find the cache filename of chart '%s'", st->id);
} }
rrddim_foreach_read(rd, st) { rrddim_foreach_read(rd, st) {
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || rd->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) { const char *cache_filename = rrddim_cache_filename(rd);
info("Deleting dimension file '%s'.", rd->cache_filename); if(!cache_filename) continue;
if(unlikely(unlink(rd->cache_filename) == -1))
error("Cannot delete dimension file '%s'", rd->cache_filename); info("Deleting dimension file '%s'.", cache_filename);
} if(unlikely(unlink(cache_filename) == -1))
error("Cannot delete dimension file '%s'", cache_filename);
} }
recursively_delete_dir(st->cache_dir, "left-over chart"); recursively_delete_dir(st->cache_dir, "left-over chart");
@ -473,11 +456,11 @@ void rrdset_delete_obsolete_dimensions(RRDSET *st) {
rrddim_foreach_read(rd, st) { rrddim_foreach_read(rd, st) {
if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || rd->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) { const char *cache_filename = rrddim_cache_filename(rd);
info("Deleting dimension file '%s'.", rd->cache_filename); if(!cache_filename) continue;
if(unlikely(unlink(rd->cache_filename) == -1)) info("Deleting dimension file '%s'.", cache_filename);
error("Cannot delete dimension file '%s'", rd->cache_filename); if(unlikely(unlink(cache_filename) == -1))
} error("Cannot delete dimension file '%s'", cache_filename);
} }
} }
} }
@ -699,8 +682,6 @@ RRDSET *rrdset_create_custom(
return st; return st;
} }
char fullfilename[FILENAME_MAX + 1];
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// get the options from the config, we need to create it // get the options from the config, we need to create it
@ -708,126 +689,37 @@ RRDSET *rrdset_create_custom(
if (memory_mode != RRD_MEMORY_MODE_DBENGINE) if (memory_mode != RRD_MEMORY_MODE_DBENGINE)
entries = align_entries_to_pagesize(memory_mode, history_entries); entries = align_entries_to_pagesize(memory_mode, history_entries);
unsigned long size = sizeof(RRDSET);
char *cache_dir = rrdset_cache_dir(host, fullid); char *cache_dir = rrdset_cache_dir(host, fullid);
time_t now = now_realtime_sec();
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// load it or allocate it // load it or allocate it
debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id); debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir); st = callocz(1, sizeof(RRDSET));
if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP || st->state = callocz(1, sizeof(*st->state));
memory_mode == RRD_MEMORY_MODE_RAM) {
st = (RRDSET *)netdata_mmap(
(memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename,
size,
((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
0);
if(st) {
memset(&st->avl, 0, sizeof(avl_t));
memset(&st->avlname, 0, sizeof(avl_t));
memset(&st->rrdvar_root_index, 0, sizeof(avl_tree_lock));
memset(&st->dimensions_index, 0, sizeof(avl_tree_lock));
memset(&st->rrdset_rwlock, 0, sizeof(netdata_rwlock_t));
st->name = NULL;
st->type = NULL;
st->family = NULL;
st->title = NULL;
st->units = NULL;
st->context = NULL;
st->cache_dir = NULL;
st->plugin_name = NULL;
st->module_name = NULL;
st->dimensions = NULL;
st->rrdfamily = NULL;
st->rrdhost = NULL;
st->next = NULL;
st->variables = NULL;
st->alarms = NULL;
st->flags = 0x00000000;
st->exporting_flags = NULL;
if(memory_mode == RRD_MEMORY_MODE_RAM) {
memset(st, 0, size);
}
else {
if(strcmp(st->magic, RRDSET_MAGIC) != 0) {
info("Initializing file %s.", fullfilename);
memset(st, 0, size);
}
else if(strcmp(st->id, fullid) != 0) {
error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid);
// munmap(st, size);
// st = NULL;
memset(st, 0, size);
}
else if(st->memsize != size || st->entries != entries) {
error("File %s does not have the desired size. Clearing it.", fullfilename);
memset(st, 0, size);
}
else if(st->update_every != update_every) {
error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
memset(st, 0, size);
}
else if((now - st->last_updated.tv_sec) > update_every * entries) {
info("File %s is too old. Clearing it.", fullfilename);
memset(st, 0, size);
}
else if(st->last_updated.tv_sec > now + update_every) {
error("File %s refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st->last_updated.tv_sec - now));
st->last_updated.tv_sec = now;
}
// make sure the database is aligned
if(st->last_updated.tv_sec) {
st->update_every = update_every;
last_updated_time_align(st);
}
}
// make sure we have the right memory mode
// even if we cleared the memory
st->rrd_memory_mode = memory_mode;
}
}
if(unlikely(!st)) {
st = callocz(1, size);
if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
st->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
else
st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
st->plugin_name = plugin?strdupz(plugin):NULL;
st->module_name = module?strdupz(module):NULL;
st->rrdhost = host;
st->memsize = size;
st->entries = entries;
st->update_every = update_every;
if(st->current_entry >= st->entries) st->current_entry = 0;
strcpy(st->cache_filename, fullfilename);
strcpy(st->magic, RRDSET_MAGIC);
strcpy(st->id, fullid); strcpy(st->id, fullid);
st->hash = simple_hash(st->id); st->hash = simple_hash(st->id);
st->rrdhost = host;
st->cache_dir = cache_dir; st->cache_dir = cache_dir;
st->entries = entries;
st->update_every = update_every;
st->chart_type = chart_type; if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP) {
st->type = strdupz(type); if(!rrdset_memory_load_or_create_map_save(st, memory_mode)) {
info("Failed to use memory mode %s for chart '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name);
memory_mode = RRD_MEMORY_MODE_RAM;
}
}
st->rrd_memory_mode = memory_mode;
st->state = callocz(1, sizeof(*st->state)); st->plugin_name = plugin?strdupz(plugin):NULL;
st->module_name = module?strdupz(module):NULL;
st->family = family ? strdupz(family) : strdupz(st->type); st->chart_type = chart_type;
st->type = strdupz(type);
st->family = family ? strdupz(family) : strdupz(st->type);
json_fix_string(st->family); json_fix_string(st->family);
st->state->is_ar_chart = strcmp(st->id, ML_ANOMALY_RATES_CHART_ID) == 0; st->state->is_ar_chart = strcmp(st->id, ML_ANOMALY_RATES_CHART_ID) == 0;
@ -848,16 +740,8 @@ RRDSET *rrdset_create_custom(
st->green = NAN; st->green = NAN;
st->red = NAN; st->red = NAN;
st->last_collected_time.tv_sec = 0;
st->last_collected_time.tv_usec = 0;
st->counter_done = 0;
st->rrddim_page_alignment = 0;
st->gap_when_lost_iterations_above = (int) (gap_when_lost_iterations_above + 2); st->gap_when_lost_iterations_above = (int) (gap_when_lost_iterations_above + 2);
st->last_accessed_time = 0;
st->upstream_resync_time = 0;
avl_init_lock(&st->dimensions_index, rrddim_compare); avl_init_lock(&st->dimensions_index, rrddim_compare);
avl_init_lock(&st->rrdvar_root_index, rrdvar_compare); avl_init_lock(&st->rrdvar_root_index, rrdvar_compare);
@ -963,7 +847,8 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
if(unlikely(since_last_usec < 0)) { if(unlikely(since_last_usec < 0)) {
// oops! the database is in the future // oops! the database is in the future
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
info("RRD database for chart '%s' on host '%s' is %0.5" LONG_DOUBLE_MODIFIER " secs in the future (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (LONG_DOUBLE)-since_last_usec / USEC_PER_SEC, st->counter, st->counter_done); info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
" secs in the future (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
#endif #endif
st->last_collected_time.tv_sec = now.tv_sec - st->update_every; st->last_collected_time.tv_sec = now.tv_sec - st->update_every;
@ -982,7 +867,8 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 5 * USEC_PER_SEC))) { else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 5 * USEC_PER_SEC))) {
// oops! the database is too far behind // oops! the database is too far behind
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
info("RRD database for chart '%s' on host '%s' is %0.5" LONG_DOUBLE_MODIFIER " secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (LONG_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done); info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
" secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
#endif #endif
microseconds = (usec_t)since_last_usec; microseconds = (usec_t)since_last_usec;
@ -1039,7 +925,7 @@ static inline usec_t rrdset_init_last_collected_time(RRDSET *st) {
usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec; usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "initialized last collected time to %0.3" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)last_collect_ut / USEC_PER_SEC); rrdset_debug(st, "initialized last collected time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_collect_ut / USEC_PER_SEC);
#endif #endif
return last_collect_ut; return last_collect_ut;
@ -1052,7 +938,7 @@ static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC); st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC);
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "updated last collected time to %0.3" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)last_collect_ut / USEC_PER_SEC); rrdset_debug(st, "updated last collected time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_collect_ut / USEC_PER_SEC);
#endif #endif
return last_collect_ut; return last_collect_ut;
@ -1071,7 +957,7 @@ static inline usec_t rrdset_init_last_updated_time(RRDSET *st) {
usec_t last_updated_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec; usec_t last_updated_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "initialized last updated time to %0.3" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)last_updated_ut / USEC_PER_SEC); rrdset_debug(st, "initialized last updated time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_updated_ut / USEC_PER_SEC);
#endif #endif
return last_updated_ut; return last_updated_ut;
@ -1109,8 +995,8 @@ static inline size_t rrdset_done_interpolate(
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); } if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
rrdset_debug(st, "last_stored_ut = %0.3" LONG_DOUBLE_MODIFIER " (last updated time)", (LONG_DOUBLE)last_stored_ut/USEC_PER_SEC); rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
rrdset_debug(st, "next_store_ut = %0.3" LONG_DOUBLE_MODIFIER " (next interpolation point)", (LONG_DOUBLE)next_store_ut/USEC_PER_SEC); rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
#endif #endif
last_ut = next_store_ut; last_ut = next_store_ut;
@ -1119,20 +1005,19 @@ static inline size_t rrdset_done_interpolate(
if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
continue; continue;
calculated_number new_value; NETDATA_DOUBLE new_value;
switch(rd->algorithm) { switch(rd->algorithm) {
case RRD_ALGORITHM_INCREMENTAL: case RRD_ALGORITHM_INCREMENTAL:
new_value = (calculated_number) new_value = (NETDATA_DOUBLE)
( rd->calculated_value ( rd->calculated_value
* (calculated_number)(next_store_ut - last_collect_ut) * (NETDATA_DOUBLE)(next_store_ut - last_collect_ut)
/ (calculated_number)(now_collect_ut - last_collect_ut) / (NETDATA_DOUBLE)(now_collect_ut - last_collect_ut)
); );
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC2 INC " rrdset_debug(st, "%s: CALC2 INC " NETDATA_DOUBLE_FORMAT " = "
CALCULATED_NUMBER_FORMAT " = " NETDATA_DOUBLE_FORMAT
CALCULATED_NUMBER_FORMAT
" * (%llu - %llu)" " * (%llu - %llu)"
" / (%llu - %llu)" " / (%llu - %llu)"
, rd->name , rd->name
@ -1146,18 +1031,18 @@ static inline size_t rrdset_done_interpolate(
rd->calculated_value -= new_value; rd->calculated_value -= new_value;
new_value += rd->last_calculated_value; new_value += rd->last_calculated_value;
rd->last_calculated_value = 0; rd->last_calculated_value = 0;
new_value /= (calculated_number)st->update_every; new_value /= (NETDATA_DOUBLE)st->update_every;
if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) { if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING", rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " NETDATA_DOUBLE_FORMAT " - EXTRAPOLATING",
rd->name rd->name
, (calculated_number)(next_store_ut - last_stored_ut) , (NETDATA_DOUBLE)(next_store_ut - last_stored_ut)
); );
#endif #endif
new_value = new_value * (calculated_number)(st->update_every * USEC_PER_SEC) / (calculated_number)(next_store_ut - last_stored_ut); new_value = new_value * (NETDATA_DOUBLE)(st->update_every * USEC_PER_SEC) / (NETDATA_DOUBLE)(next_store_ut - last_stored_ut);
} }
break; break;
@ -1176,21 +1061,19 @@ static inline size_t rrdset_done_interpolate(
// we have missed an update // we have missed an update
// interpolate in the middle values // interpolate in the middle values
new_value = (calculated_number) new_value = (NETDATA_DOUBLE)
( ( (rd->calculated_value - rd->last_calculated_value) ( ( (rd->calculated_value - rd->last_calculated_value)
* (calculated_number)(next_store_ut - last_collect_ut) * (NETDATA_DOUBLE)(next_store_ut - last_collect_ut)
/ (calculated_number)(now_collect_ut - last_collect_ut) / (NETDATA_DOUBLE)(now_collect_ut - last_collect_ut)
) )
+ rd->last_calculated_value + rd->last_calculated_value
); );
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC2 DEF " rrdset_debug(st, "%s: CALC2 DEF " NETDATA_DOUBLE_FORMAT " = ((("
CALCULATED_NUMBER_FORMAT " = (((" "(" NETDATA_DOUBLE_FORMAT " - " NETDATA_DOUBLE_FORMAT ")"
"(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
" * %llu" " * %llu"
" / %llu) + " CALCULATED_NUMBER_FORMAT " / %llu) + " NETDATA_DOUBLE_FORMAT, rd->name
, rd->name
, new_value , new_value
, rd->calculated_value, rd->last_calculated_value , rd->calculated_value, rd->last_calculated_value
, (next_store_ut - first_ut) , (next_store_ut - first_ut)
@ -1271,7 +1154,7 @@ static inline void rrdset_done_fill_the_gap(RRDSET *st) {
long current_entry = st->current_entry; long current_entry = st->current_entry;
for(c = 0; c < entries && next_store_ut <= now_collect_ut ; next_store_ut += update_every_ut, c++) { for(c = 0; c < entries && next_store_ut <= now_collect_ut ; next_store_ut += update_every_ut, c++) {
rd->values[current_entry] = SN_EMPTY_SLOT; rd->db[current_entry] = SN_EMPTY_SLOT;
current_entry = ((current_entry + 1) >= entries) ? 0 : current_entry + 1; current_entry = ((current_entry + 1) >= entries) ? 0 : current_entry + 1;
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
@ -1332,7 +1215,8 @@ void rrdset_done(RRDSET *st) {
// check if the chart has a long time to be updated // check if the chart has a long time to be updated
if(unlikely(st->usec_since_last_update > st->entries * update_every_ut && if(unlikely(st->usec_since_last_update > st->entries * update_every_ut &&
st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && st->rrd_memory_mode != RRD_MEMORY_MODE_NONE)) { st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && st->rrd_memory_mode != RRD_MEMORY_MODE_NONE)) {
info("host '%s', chart %s: took too long to be updated (counter #%zu, update #%zu, %0.3" LONG_DOUBLE_MODIFIER " secs). Resetting it.", st->rrdhost->hostname, st->name, st->counter, st->counter_done, (LONG_DOUBLE)st->usec_since_last_update / USEC_PER_SEC); info("host '%s', chart %s: took too long to be updated (counter #%zu, update #%zu, %0.3" NETDATA_DOUBLE_MODIFIER
" secs). Resetting it.", st->rrdhost->hostname, st->name, st->counter, st->counter_done, (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);
rrdset_reset(st); rrdset_reset(st);
st->usec_since_last_update = update_every_ut; st->usec_since_last_update = update_every_ut;
store_this_entry = 0; store_this_entry = 0;
@ -1459,6 +1343,7 @@ void rrdset_done(RRDSET *st) {
#endif #endif
} }
} }
after_first_database_work: after_first_database_work:
st->counter_done++; st->counter_done++;
@ -1469,10 +1354,10 @@ after_first_database_work:
} }
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "last_collect_ut = %0.3" LONG_DOUBLE_MODIFIER " (last collection time)", (LONG_DOUBLE)last_collect_ut/USEC_PER_SEC); rrdset_debug(st, "last_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last collection time)", (NETDATA_DOUBLE)last_collect_ut/USEC_PER_SEC);
rrdset_debug(st, "now_collect_ut = %0.3" LONG_DOUBLE_MODIFIER " (current collection time)", (LONG_DOUBLE)now_collect_ut/USEC_PER_SEC); rrdset_debug(st, "now_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (current collection time)", (NETDATA_DOUBLE)now_collect_ut/USEC_PER_SEC);
rrdset_debug(st, "last_stored_ut = %0.3" LONG_DOUBLE_MODIFIER " (last updated time)", (LONG_DOUBLE)last_stored_ut/USEC_PER_SEC); rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
rrdset_debug(st, "next_store_ut = %0.3" LONG_DOUBLE_MODIFIER " (next interpolation point)", (LONG_DOUBLE)next_store_ut/USEC_PER_SEC); rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
#endif #endif
// calculate totals and count the dimensions // calculate totals and count the dimensions
@ -1481,7 +1366,9 @@ after_first_database_work:
rrddim_foreach_read(rd, st) { rrddim_foreach_read(rd, st) {
if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
continue; continue;
dimensions++; dimensions++;
if(likely(rd->updated)) if(likely(rd->updated))
st->collected_total += rd->collected_value; st->collected_total += rd->collected_value;
} }
@ -1509,9 +1396,8 @@ after_first_database_work:
rrdset_debug(st, "%s: START " rrdset_debug(st, "%s: START "
" last_collected_value = " COLLECTED_NUMBER_FORMAT " last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " CALCULATED_NUMBER_FORMAT " last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " CALCULATED_NUMBER_FORMAT " calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
, rd->name
, rd->last_collected_value , rd->last_collected_value
, rd->collected_value , rd->collected_value
, rd->last_calculated_value , rd->last_calculated_value
@ -1521,21 +1407,19 @@ after_first_database_work:
switch(rd->algorithm) { switch(rd->algorithm) {
case RRD_ALGORITHM_ABSOLUTE: case RRD_ALGORITHM_ABSOLUTE:
rd->calculated_value = (calculated_number)rd->collected_value rd->calculated_value = (NETDATA_DOUBLE)rd->collected_value
* (calculated_number)rd->multiplier * (NETDATA_DOUBLE)rd->multiplier
/ (calculated_number)rd->divisor; / (NETDATA_DOUBLE)rd->divisor;
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN " rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN " NETDATA_DOUBLE_FORMAT " = "
CALCULATED_NUMBER_FORMAT " = "
COLLECTED_NUMBER_FORMAT COLLECTED_NUMBER_FORMAT
" * " CALCULATED_NUMBER_FORMAT " * " NETDATA_DOUBLE_FORMAT
" / " CALCULATED_NUMBER_FORMAT " / " NETDATA_DOUBLE_FORMAT, rd->name
, rd->name
, rd->calculated_value , rd->calculated_value
, rd->collected_value , rd->collected_value
, (calculated_number)rd->multiplier , (NETDATA_DOUBLE)rd->multiplier
, (calculated_number)rd->divisor , (NETDATA_DOUBLE)rd->divisor
); );
#endif #endif
@ -1548,13 +1432,12 @@ after_first_database_work:
// the percentage of the current value // the percentage of the current value
// over the total of all dimensions // over the total of all dimensions
rd->calculated_value = rd->calculated_value =
(calculated_number)100 (NETDATA_DOUBLE)100
* (calculated_number)rd->collected_value * (NETDATA_DOUBLE)rd->collected_value
/ (calculated_number)st->collected_total; / (NETDATA_DOUBLE)st->collected_total;
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC PCENT-ROW " rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100"
CALCULATED_NUMBER_FORMAT " = 100"
" * " COLLECTED_NUMBER_FORMAT " * " COLLECTED_NUMBER_FORMAT
" / " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT
, rd->name , rd->name
@ -1605,34 +1488,32 @@ after_first_database_work:
// TODO: remember recent history of rates and compare with current rate to reduce this chance. // TODO: remember recent history of rates and compare with current rate to reduce this chance.
if (delta < max_acceptable_rate) { if (delta < max_acceptable_rate) {
rd->calculated_value += rd->calculated_value +=
(calculated_number) delta (NETDATA_DOUBLE) delta
* (calculated_number) rd->multiplier * (NETDATA_DOUBLE) rd->multiplier
/ (calculated_number) rd->divisor; / (NETDATA_DOUBLE) rd->divisor;
} else { } else {
// This is a reset. Any overflow with a rate greater than MAX_INCREMENTAL_PERCENT_RATE will also // This is a reset. Any overflow with a rate greater than MAX_INCREMENTAL_PERCENT_RATE will also
// be detected as a reset instead. // be detected as a reset instead.
rd->calculated_value += (calculated_number)0; rd->calculated_value += (NETDATA_DOUBLE)0;
} }
} }
else { else {
rd->calculated_value += rd->calculated_value +=
(calculated_number) (rd->collected_value - rd->last_collected_value) (NETDATA_DOUBLE) (rd->collected_value - rd->last_collected_value)
* (calculated_number) rd->multiplier * (NETDATA_DOUBLE) rd->multiplier
/ (calculated_number) rd->divisor; / (NETDATA_DOUBLE) rd->divisor;
} }
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC INC PRE " rrdset_debug(st, "%s: CALC INC PRE " NETDATA_DOUBLE_FORMAT " = ("
CALCULATED_NUMBER_FORMAT " = ("
COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
")" ")"
" * " CALCULATED_NUMBER_FORMAT " * " NETDATA_DOUBLE_FORMAT
" / " CALCULATED_NUMBER_FORMAT " / " NETDATA_DOUBLE_FORMAT, rd->name
, rd->name
, rd->calculated_value , rd->calculated_value
, rd->collected_value, rd->last_collected_value , rd->collected_value, rd->last_collected_value
, (calculated_number)rd->multiplier , (NETDATA_DOUBLE)rd->multiplier
, (calculated_number)rd->divisor , (NETDATA_DOUBLE)rd->divisor
); );
#endif #endif
@ -1665,13 +1546,12 @@ after_first_database_work:
rd->calculated_value = 0; rd->calculated_value = 0;
else else
rd->calculated_value = rd->calculated_value =
(calculated_number)100 (NETDATA_DOUBLE)100
* (calculated_number)(rd->collected_value - rd->last_collected_value) * (NETDATA_DOUBLE)(rd->collected_value - rd->last_collected_value)
/ (calculated_number)(st->collected_total - st->last_collected_total); / (NETDATA_DOUBLE)(st->collected_total - st->last_collected_total);
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC PCENT-DIFF " rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100"
CALCULATED_NUMBER_FORMAT " = 100"
" * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
" / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
, rd->name , rd->name
@ -1689,8 +1569,7 @@ after_first_database_work:
rd->calculated_value = 0; rd->calculated_value = 0;
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC " rrdset_debug(st, "%s: CALC " NETDATA_DOUBLE_FORMAT " = 0"
CALCULATED_NUMBER_FORMAT " = 0"
, rd->name , rd->name
, rd->calculated_value , rd->calculated_value
); );
@ -1703,9 +1582,8 @@ after_first_database_work:
rrdset_debug(st, "%s: PHASE2 " rrdset_debug(st, "%s: PHASE2 "
" last_collected_value = " COLLECTED_NUMBER_FORMAT " last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " CALCULATED_NUMBER_FORMAT " last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " CALCULATED_NUMBER_FORMAT " calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
, rd->name
, rd->last_collected_value , rd->last_collected_value
, rd->collected_value , rd->collected_value
, rd->last_calculated_value , rd->last_calculated_value
@ -1765,7 +1643,8 @@ after_second_database_work:
case RRD_ALGORITHM_INCREMENTAL: case RRD_ALGORITHM_INCREMENTAL:
if(unlikely(!first_entry)) { if(unlikely(!first_entry)) {
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value); rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT
") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
#endif #endif
rd->last_calculated_value += rd->calculated_value; rd->last_calculated_value += rd->calculated_value;
@ -1781,7 +1660,8 @@ after_second_database_work:
case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value); rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT
") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value);
#endif #endif
rd->last_calculated_value = rd->calculated_value; rd->last_calculated_value = rd->calculated_value;
@ -1796,9 +1676,8 @@ after_second_database_work:
rrdset_debug(st, "%s: END " rrdset_debug(st, "%s: END "
" last_collected_value = " COLLECTED_NUMBER_FORMAT " last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " CALCULATED_NUMBER_FORMAT " last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " CALCULATED_NUMBER_FORMAT " calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
, rd->name
, rd->last_collected_value , rd->last_collected_value
, rd->collected_value , rd->collected_value
, rd->last_calculated_value , rd->last_calculated_value
@ -1811,15 +1690,24 @@ after_second_database_work:
// ALL DONE ABOUT THE DATA UPDATE // ALL DONE ABOUT THE DATA UPDATE
// -------------------------------------------------------------------- // --------------------------------------------------------------------
// find if there are any obsolete dimensions if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
time_t now = now_realtime_sec(); // update the memory mapped files with the latest values
rrdset_memory_file_update(st);
rrddim_foreach_read(rd, st) {
rrddim_memory_file_update(rd);
}
}
// find if there are any obsolete dimensions
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) { if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) {
rrddim_foreach_read(rd, st) rrddim_foreach_read(rd, st)
if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)))
break; break;
if(unlikely(rd)) { if(unlikely(rd)) {
time_t now = now_realtime_sec();
RRDDIM *last; RRDDIM *last;
// there is a dimension to free // there is a dimension to free
// upgrade our read lock to a write lock // upgrade our read lock to a write lock
@ -1831,10 +1719,11 @@ after_second_database_work:
&& (rd->last_collected_time.tv_sec + rrdset_free_obsolete_time < now))) { && (rd->last_collected_time.tv_sec + rrdset_free_obsolete_time < now))) {
info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id); info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || rd->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) { const char *cache_filename = rrddim_cache_filename(rd);
info("Deleting dimension file '%s'.", rd->cache_filename); if(cache_filename) {
if(unlikely(unlink(rd->cache_filename) == -1)) info("Deleting dimension file '%s'.", cache_filename);
error("Cannot delete dimension file '%s'", rd->cache_filename); if (unlikely(unlink(cache_filename) == -1))
error("Cannot delete dimension file '%s'", cache_filename);
} }
#ifdef ENABLE_DBENGINE #ifdef ENABLE_DBENGINE
@ -1885,3 +1774,208 @@ after_second_database_work:
netdata_thread_enable_cancelability(); netdata_thread_enable_cancelability();
} }
// ----------------------------------------------------------------------------
// compatibility layer for RRDSET files v019
#define RRDSET_MAGIC_V019 "NETDATA RRD SET FILE V019"
#define RRD_ID_LENGTH_MAX_V019 200
struct avl_element_v019 {
void *avl_link[2];
signed char avl_balance;
};
struct avl_tree_type_v019 {
void *root;
int (*compar)(void *a, void *b);
};
struct avl_tree_lock_v019 {
struct avl_tree_type_v019 avl_tree;
pthread_rwlock_t rwlock;
};
struct rrdset_map_save_v019 {
struct avl_element_v019 avl; // ignored
struct avl_element_v019 avlname; // ignored
char id[RRD_ID_LENGTH_MAX_V019 + 1]; // check to reset all - update on load
void *name; // ignored
void *unused_ptr; // ignored
void *type; // ignored
void *family; // ignored
void *title; // ignored
void *units; // ignored
void *context; // ignored
uint32_t hash_context; // ignored
uint32_t chart_type; // ignored
int update_every; // check to reset all - update on load
long entries; // check to reset all - update on load
long current_entry; // NEEDS TO BE UPDATED - FIXED ON LOAD
uint32_t flags; // ignored
void *exporting_flags; // ignored
int gap_when_lost_iterations_above; // ignored
long priority; // ignored
uint32_t rrd_memory_mode; // ignored
void *cache_dir; // ignored
char cache_filename[FILENAME_MAX+1]; // ignored - update on load
pthread_rwlock_t rrdset_rwlock; // ignored
size_t counter; // NEEDS TO BE UPDATED - maintained on load
size_t counter_done; // ignored
union { //
time_t last_accessed_time; // ignored
time_t last_entry_t; // ignored
}; //
time_t upstream_resync_time; // ignored
void *plugin_name; // ignored
void *module_name; // ignored
void *chart_uuid; // ignored
void *state; // ignored
size_t unused[3]; // ignored
size_t rrddim_page_alignment; // ignored
uint32_t hash; // ignored
uint32_t hash_name; // ignored
usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load
struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load
struct timeval last_collected_time; // ignored
long long collected_total; // NEEDS TO BE UPDATED - maintained on load
long long last_collected_total; // NEEDS TO BE UPDATED - maintained on load
void *rrdfamily; // ignored
void *rrdhost; // ignored
void *next; // ignored
long double green; // ignored
long double red; // ignored
struct avl_tree_lock_v019 rrdvar_root_index; // ignored
void *variables; // ignored
void *alarms; // ignored
unsigned long memsize; // check to reset all - update on load
char magic[sizeof(RRDSET_MAGIC_V019) + 1]; // check to reset all - update on load
struct avl_tree_lock_v019 dimensions_index; // ignored
void *dimensions; // ignored
};
void rrdset_memory_file_update(RRDSET *st) {
if(!st->st_on_file) return;
struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
st_on_file->current_entry = st->current_entry;
st_on_file->counter = st->counter;
st_on_file->usec_since_last_update = st->usec_since_last_update;
st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
st_on_file->collected_total = st->collected_total;
st_on_file->last_collected_total = st->last_collected_total;
}
const char *rrdset_cache_filename(RRDSET *st) {
if(!st->st_on_file) return NULL;
struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
return st_on_file->cache_filename;
}
void rrdset_memory_file_free(RRDSET *st) {
if(!st->st_on_file) return;
// needed for memory mode map, to save the latest state
rrdset_memory_file_update(st);
struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
munmap(st_on_file, st_on_file->memsize);
// remove the pointers from the RRDDIM
st->st_on_file = NULL;
}
void rrdset_memory_file_save(RRDSET *st) {
if(!st->st_on_file) return;
rrdset_memory_file_update(st);
struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
if(st_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
memory_file_save(st_on_file->cache_filename, st->st_on_file, st_on_file->memsize);
}
bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mode) {
if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
return false;
char fullfilename[FILENAME_MAX + 1];
snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", st->cache_dir);
unsigned long size = sizeof(struct rrdset_map_save_v019);
struct rrdset_map_save_v019 *st_on_file = (struct rrdset_map_save_v019 *)netdata_mmap(
fullfilename, size,
((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
0);
if(!st_on_file) return false;
time_t now = now_realtime_sec();
if(strcmp(st_on_file->magic, RRDSET_MAGIC_V019) != 0) {
info("Initializing file '%s'.", fullfilename);
memset(st_on_file, 0, size);
}
else if(strncmp(st_on_file->id, st->id, RRD_ID_LENGTH_MAX_V019) != 0) {
error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, st->id);
memset(st_on_file, 0, size);
}
else if(st_on_file->memsize != size || st_on_file->entries != st->entries) {
error("File '%s' does not have the desired size. Clearing it.", fullfilename);
memset(st_on_file, 0, size);
}
else if(st_on_file->update_every != st->update_every) {
error("File '%s' does not have the desired granularity. Clearing it.", fullfilename);
memset(st_on_file, 0, size);
}
else if((now - st_on_file->last_updated.tv_sec) > st->update_every * st->entries) {
info("File '%s' is too old. Clearing it.", fullfilename);
memset(st_on_file, 0, size);
}
else if(st_on_file->last_updated.tv_sec > now + st->update_every) {
error("File '%s' refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st_on_file->last_updated.tv_sec - now));
st_on_file->last_updated.tv_sec = now;
}
if(st_on_file->current_entry >= st_on_file->entries)
st_on_file->current_entry = 0;
// make sure the database is aligned
bool align_last_updated = false;
if(st_on_file->last_updated.tv_sec) {
st_on_file->update_every = st->update_every;
align_last_updated = true;
}
// copy the useful values to st
st->current_entry = st_on_file->current_entry;
st->counter = st_on_file->counter;
st->usec_since_last_update = st_on_file->usec_since_last_update;
st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
st->collected_total = st_on_file->collected_total;
st->last_collected_total = st_on_file->last_collected_total;
// link it to st
st->st_on_file = st_on_file;
// clear everything
memset(st_on_file, 0, size);
// set the values we need
strncpyz(st_on_file->id, st->id, RRD_ID_LENGTH_MAX_V019 + 1);
strcpy(st_on_file->cache_filename, fullfilename);
strcpy(st_on_file->magic, RRDSET_MAGIC_V019);
st_on_file->memsize = size;
st_on_file->entries = st->entries;
st_on_file->update_every = st->update_every;
st_on_file->rrd_memory_mode = memory_mode;
if(align_last_updated)
last_updated_time_align(st);
// copy the useful values back to st_on_file
rrdset_memory_file_update(st);
return true;
}

9
database/rrdsetvar.c
View File

@ -163,7 +163,7 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
// not found, allocate one // not found, allocate one
calculated_number *v = mallocz(sizeof(calculated_number)); NETDATA_DOUBLE *v = mallocz(sizeof(NETDATA_DOUBLE));
*v = NAN; *v = NAN;
rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR); rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR);
@ -173,12 +173,13 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
return rs; return rs;
} }
void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, calculated_number value) { void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, NETDATA_DOUBLE value) {
if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) || !(rs->options & RRDVAR_OPTION_ALLOCATED)) { if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) || !(rs->options & RRDVAR_OPTION_ALLOCATED)) {
error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom chart one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value); error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " NETDATA_DOUBLE_FORMAT
" but the variable is not a custom chart one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value);
} }
else { else {
calculated_number *v = rs->value; NETDATA_DOUBLE *v = rs->value;
if(*v != value) { if(*v != value) {
*v = value; *v = value;

2
database/rrdsetvar.h
View File

@ -35,7 +35,7 @@ struct rrdsetvar {
}; };
extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name); extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name);
extern void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rv, calculated_number value); extern void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rv, NETDATA_DOUBLE value);
extern void rrdsetvar_rename_all(RRDSET *st); extern void rrdsetvar_rename_all(RRDSET *st);
extern RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options); extern RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options);

20
database/rrdvar.c
View File

@ -133,7 +133,7 @@ inline int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)
} }
static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *tree_lock, const char *name) { static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *tree_lock, const char *name) {
calculated_number *v = callocz(1, sizeof(calculated_number)); NETDATA_DOUBLE *v = callocz(1, sizeof(NETDATA_DOUBLE));
*v = NAN; *v = NAN;
RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v); RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v);
@ -158,11 +158,11 @@ RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) {
return rrdvar_custom_variable_create("host", &host->rrdvar_root_index, name); return rrdvar_custom_variable_create("host", &host->rrdvar_root_index, name);
} }
void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value) { void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value) {
if(rv->type != RRDVAR_TYPE_CALCULATED || !(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR) || !(rv->options & RRDVAR_OPTION_ALLOCATED)) if(rv->type != RRDVAR_TYPE_CALCULATED || !(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR) || !(rv->options & RRDVAR_OPTION_ALLOCATED))
error("requested to set variable '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rv->name, value); error("requested to set variable '%s' to value " NETDATA_DOUBLE_FORMAT " but the variable is not a custom one.", rv->name, value);
else { else {
calculated_number *v = rv->value; NETDATA_DOUBLE *v = rv->value;
if(*v != value) { if(*v != value) {
*v = value; *v = value;
@ -181,10 +181,10 @@ int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR * /*rv*
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// RRDVAR lookup // RRDVAR lookup
calculated_number rrdvar2number(RRDVAR *rv) { NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) {
switch(rv->type) { switch(rv->type) {
case RRDVAR_TYPE_CALCULATED: { case RRDVAR_TYPE_CALCULATED: {
calculated_number *n = (calculated_number *)rv->value; NETDATA_DOUBLE *n = (NETDATA_DOUBLE *)rv->value;
return *n; return *n;
} }
@ -209,12 +209,12 @@ calculated_number rrdvar2number(RRDVAR *rv) {
} }
default: default:
error("I don't know how to convert RRDVAR type %u to calculated_number", rv->type); error("I don't know how to convert RRDVAR type %u to NETDATA_DOUBLE", rv->type);
return NAN; return NAN;
} }
} }
int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result) { int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, NETDATA_DOUBLE *result) {
RRDSET *st = rc->rrdset; RRDSET *st = rc->rrdset;
if(!st) return 0; if(!st) return 0;
@ -254,13 +254,13 @@ struct variable2json_helper {
static int single_variable2json(void *entry, void *data) { static int single_variable2json(void *entry, void *data) {
struct variable2json_helper *helper = (struct variable2json_helper *)data; struct variable2json_helper *helper = (struct variable2json_helper *)data;
RRDVAR *rv = (RRDVAR *)entry; RRDVAR *rv = (RRDVAR *)entry;
calculated_number value = rrdvar2number(rv); NETDATA_DOUBLE value = rrdvar2number(rv);
if (helper->options == RRDVAR_OPTION_DEFAULT || rv->options & helper->options) { if (helper->options == RRDVAR_OPTION_DEFAULT || rv->options & helper->options) {
if(unlikely(isnan(value) || isinf(value))) if(unlikely(isnan(value) || isinf(value)))
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rv->name); buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rv->name);
else else
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" LONG_DOUBLE_MODIFIER, helper->counter?",":"", rv->name, (LONG_DOUBLE)value); buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" NETDATA_DOUBLE_MODIFIER, helper->counter?",":"", rv->name, (NETDATA_DOUBLE)value);
helper->counter++; helper->counter++;
} }

4
database/rrdvar.h
View File

@ -52,13 +52,13 @@ extern int rrdvar_fix_name(char *variable);
#include "rrd.h" #include "rrd.h"
extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name); extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name);
extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value); extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value);
extern int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data); extern int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data);
extern void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock); extern void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock);
extern int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data); extern int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data);
extern calculated_number rrdvar2number(RRDVAR *rv); extern NETDATA_DOUBLE rrdvar2number(RRDVAR *rv);
extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value); extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value);
extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv); extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv);

8
database/sqlite/sqlite_aclk_alert.c
View File

@ -336,8 +336,8 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
strdupz((char *)format_value_and_unit( strdupz((char *)format_value_and_unit(
old_value_string, 100, sqlite3_column_double(res, 24), (char *)sqlite3_column_text(res, 17), -1)); old_value_string, 100, sqlite3_column_double(res, 24), (char *)sqlite3_column_text(res, 17), -1));
alarm_log.value = (calculated_number) sqlite3_column_double(res, 23); alarm_log.value = (NETDATA_DOUBLE) sqlite3_column_double(res, 23);
alarm_log.old_value = (calculated_number) sqlite3_column_double(res, 24); alarm_log.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 24);
alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0; alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
alarm_log.rendered_info = sqlite3_column_type(res, 18) == SQLITE_NULL ? alarm_log.rendered_info = sqlite3_column_type(res, 18) == SQLITE_NULL ?
@ -868,8 +868,8 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN
alarm_log->value_string = strdupz(ae->new_value_string); alarm_log->value_string = strdupz(ae->new_value_string);
alarm_log->old_value_string = strdupz(ae->old_value_string); alarm_log->old_value_string = strdupz(ae->old_value_string);
alarm_log->value = (!isnan(ae->new_value)) ? (calculated_number)ae->new_value : 0; alarm_log->value = (!isnan(ae->new_value)) ? (NETDATA_DOUBLE)ae->new_value : 0;
alarm_log->old_value = (!isnan(ae->old_value)) ? (calculated_number)ae->old_value : 0; alarm_log->old_value = (!isnan(ae->old_value)) ? (NETDATA_DOUBLE)ae->old_value : 0;
alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0; alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
alarm_log->rendered_info = ae->info ? strdupz(ae->info) : strdupz((char *)""); alarm_log->rendered_info = ae->info ? strdupz(ae->info) : strdupz((char *)"");

4
database/sqlite/sqlite_health.c
View File

@ -751,8 +751,8 @@ void sql_health_alarm_log_load(RRDHOST *host) {
ae->old_status = (RRDCALC_STATUS)sqlite3_column_int(res, 23); ae->old_status = (RRDCALC_STATUS)sqlite3_column_int(res, 23);
ae->delay = (int) sqlite3_column_int(res, 24); ae->delay = (int) sqlite3_column_int(res, 24);
ae->new_value = (calculated_number) sqlite3_column_double(res, 25); ae->new_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 25);
ae->old_value = (calculated_number) sqlite3_column_double(res, 26); ae->old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 26);
ae->last_repeat = last_repeat; ae->last_repeat = last_repeat;

54
docs/guides/longer-metrics-storage.md
View File

@ -25,14 +25,14 @@ available disk space for long-term metrics storage. This feature of the database
larger dataset than your system's available RAM. larger dataset than your system's available RAM.
The database engine is currently the default method of storing metrics, but if you're not sure which database you're The database engine is currently the default method of storing metrics, but if you're not sure which database you're
using, check out your `netdata.conf` file and look for the `memory mode` setting: using, check out your `netdata.conf` file and look for the `[db].mode` setting:
```conf ```conf
[global] [db]
memory mode = dbengine mode = dbengine
``` ```
If `memory mode` is set to anything but `dbengine`, change it and restart Netdata using the standard command for If `[db].mode` is set to anything but `dbengine`, change it and restart Netdata using the standard command for
restarting services on your system. You're now using the database engine! restarting services on your system. You're now using the database engine!
What makes the database engine efficient? While it's structured like a traditional database, the database engine splits What makes the database engine efficient? While it's structured like a traditional database, the database engine splits
@ -43,22 +43,22 @@ When the Netdata dashboard queries for historical metrics, the database engine w
return relevant metrics for visualization in charts. return relevant metrics for visualization in charts.
Now, given that the database engine uses _both_ RAM and disk, there are two other settings to consider: `page cache Now, given that the database engine uses _both_ RAM and disk, there are two other settings to consider: `page cache
size` and `dbengine multihost disk space`. size MB` and `dbengine multihost disk space MB`.
```conf ```conf
[global] [db]
page cache size = 32 page cache size MB = 32
dbengine multihost disk space = 256 dbengine multihost disk space MB = 256
``` ```
`page cache size` sets the maximum amount of RAM (in MiB) the database engine will use for caching and indexing. `[db].page cache size MB` sets the maximum amount of RAM the database engine will use for caching and indexing.
`dbengine multihost disk space` sets the maximum disk space (again, in MiB) the database engine will use for storing `[db].dbengine multihost disk space MB` sets the maximum disk space the database engine will use for storing
compressed metrics. The default settings retain about two day's worth of metrics on a system collecting 2,000 metrics compressed metrics. The default settings retain about four day's worth of metrics on a system collecting 2,000 metrics
every second. every second.
[**See our database engine [**See our database engine
calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics) calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics)
to help you correctly set `dbengine multihost disk space` based on your needs. The calculator gives an accurate estimate to help you correctly set `[db].dbengine multihost disk space MB` based on your needs. The calculator gives an accurate estimate
based on how many child nodes you have, how many metrics your Agent collects, and more. based on how many child nodes you have, how many metrics your Agent collects, and more.
With the database engine active, you can back up your `/var/cache/netdata/dbengine/` folder to another location for With the database engine active, you can back up your `/var/cache/netdata/dbengine/` folder to another location for
@ -72,26 +72,26 @@ aren't ready to make the move.
In previous versions, Netdata used a round-robin database to store 1 hour of per-second metrics. In previous versions, Netdata used a round-robin database to store 1 hour of per-second metrics.
To see if you're still using this database, or if you would like to switch to it, open your `netdata.conf` file and see To see if you're still using this database, or if you would like to switch to it, open your `netdata.conf` file and see
if `memory mode` option is set to `save`. if `[db].mode` option is set to `save`.
```conf ```conf
[global] [db]
memory mode = save mode = save
``` ```
If `memory mode` is set to `save`, then you're using the round-robin database. If so, the `history` option is set to If `[db].mode` is set to `save`, then you're using the round-robin database. If so, the `[db].retention` option is set to
`3600`, which is the equivalent to 3,600 seconds, or one hour. `3600`, which is the equivalent to 3,600 seconds, or one hour.
To increase your historical metrics, you can increase `history` to the number of seconds you'd like to store: To increase your historical metrics, you can increase `[db].retention` to the number of seconds you'd like to store:
```conf ```conf
[global] [db]
# 2 hours = 2 * 60 * 60 = 7200 seconds # 2 hours = 2 * 60 * 60 = 7200 seconds
history = 7200 retention = 7200
# 4 hours = 4 * 60 * 60 = 14440 seconds # 4 hours = 4 * 60 * 60 = 14440 seconds
history = 14440 retention = 14440
# 24 hours = 24 * 60 * 60 = 86400 seconds # 24 hours = 24 * 60 * 60 = 86400 seconds
history = 86400 retention = 86400
``` ```
And so on. And so on.
@ -105,20 +105,10 @@ dashboard and look at the bottom-right corner of the interface. You'll find a se
On this desktop system, using a Ryzen 5 1600 and 16GB of RAM, the round-robin databases uses 25 MB of RAM to store just On this desktop system, using a Ryzen 5 1600 and 16GB of RAM, the round-robin databases uses 25 MB of RAM to store just
over an hour's worth of data for nearly 2,000 metrics. over an hour's worth of data for nearly 2,000 metrics.
To increase the `history` option, you need to edit your `netdata.conf` file and increase the `history` setting. In most
installations, you'll find it at `/etc/netdata/netdata.conf`, but some operating systems place it at
`/opt/netdata/etc/netdata/netdata.conf`.
Use `/etc/netdata/edit-config netdata.conf`, or your favorite text editor, to replace `3600` with the number of seconds
you'd like to store.
You should base this number on two things: How much history you need for your use case, and how much RAM you're willing You should base this number on two things: How much history you need for your use case, and how much RAM you're willing
to dedicate to Netdata. to dedicate to Netdata.
> Take care when you change the `history` option on production systems. Netdata is configured to stop its process if How much RAM will a longer retention use? Let's use a little math.
> your system starts running out of RAM, but you can never be too careful. Out of memory situations are very bad.
How much RAM will a longer history use? Let's use a little math.
The round-robin database needs 4 bytes for every value Netdata collects. If Netdata collects metrics every second, The round-robin database needs 4 bytes for every value Netdata collects. If Netdata collects metrics every second,
that's 4 bytes, per second, per metric. that's 4 bytes, per second, per metric.

3
exporting/exporting_engine.h
View File

@ -269,7 +269,8 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st);
extern EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options); extern EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options);
calculated_number exporting_calculate_value_from_stored_data( NETDATA_DOUBLE
exporting_calculate_value_from_stored_data(
struct instance *instance, struct instance *instance,
RRDDIM *rd, RRDDIM *rd,
time_t *last_timestamp); time_t *last_timestamp);

4
exporting/graphite/graphite.c
View File

@ -173,14 +173,14 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
RRD_ID_LENGTH_MAX); RRD_ID_LENGTH_MAX);
time_t last_t; time_t last_t;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value)) if(isnan(value))
return 0; return 0;
buffer_sprintf( buffer_sprintf(
instance->buffer, instance->buffer,
"%s.%s.%s.%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n", "%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
instance->config.prefix, instance->config.prefix,
(host == localhost) ? instance->config.hostname : host->hostname, (host == localhost) ? instance->config.hostname : host->hostname,
chart_name, chart_name,

4
exporting/json/json.c
View File

@ -224,7 +224,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
RRDHOST *host = st->rrdhost; RRDHOST *host = st->rrdhost;
time_t last_t; time_t last_t;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value)) if(isnan(value))
return 0; return 0;
@ -265,7 +265,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
"\"id\":\"%s\"," "\"id\":\"%s\","
"\"name\":\"%s\"," "\"name\":\"%s\","
"\"value\":" CALCULATED_NUMBER_FORMAT "," "\"value\":" NETDATA_DOUBLE_FORMAT ","
"\"timestamp\": %llu}", "\"timestamp\": %llu}",

8
exporting/opentsdb/opentsdb.c
View File

@ -225,14 +225,14 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
RRD_ID_LENGTH_MAX); RRD_ID_LENGTH_MAX);
time_t last_t; time_t last_t;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value)) if(isnan(value))
return 0; return 0;
buffer_sprintf( buffer_sprintf(
instance->buffer, instance->buffer,
"put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s%s\n", "put %s.%s.%s %llu " NETDATA_DOUBLE_FORMAT " host=%s%s%s%s\n",
instance->config.prefix, instance->config.prefix,
chart_name, chart_name,
dimension_name, dimension_name,
@ -368,7 +368,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
RRD_ID_LENGTH_MAX); RRD_ID_LENGTH_MAX);
time_t last_t; time_t last_t;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value)) if(isnan(value))
return 0; return 0;
@ -381,7 +381,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
"{" "{"
"\"metric\":\"%s.%s.%s\"," "\"metric\":\"%s.%s.%s\","
"\"timestamp\":%llu," "\"timestamp\":%llu,"
"\"value\":"CALCULATED_NUMBER_FORMAT"," "\"value\":" NETDATA_DOUBLE_FORMAT ","
"\"tags\":{" "\"tags\":{"
"\"host\":\"%s%s%s\"%s" "\"host\":\"%s%s%s\"%s"
"}" "}"

10
exporting/process_data.c
View File

@ -64,7 +64,7 @@ int mark_scheduled_instances(struct engine *engine)
* @param last_timestamp the timestamp that should be reported to the exporting connector instance. * @param last_timestamp the timestamp that should be reported to the exporting connector instance.
* @return Returns the value, calculated over the given period. * @return Returns the value, calculated over the given period.
*/ */
calculated_number exporting_calculate_value_from_stored_data( NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
struct instance *instance, struct instance *instance,
RRDDIM *rd, RRDDIM *rd,
time_t *last_timestamp) time_t *last_timestamp)
@ -123,15 +123,15 @@ calculated_number exporting_calculate_value_from_stored_data(
*last_timestamp = before; *last_timestamp = before;
size_t counter = 0; size_t counter = 0;
calculated_number sum = 0; NETDATA_DOUBLE sum = 0;
calculated_number value; NETDATA_DOUBLE value;
for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) { for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {
time_t curr_t, end_t; time_t curr_t, end_t;
SN_FLAGS flags; SN_FLAGS flags;
value = rd->state->query_ops.next_metric(&handle, &curr_t, &end_t, &flags); value = rd->state->query_ops.next_metric(&handle, &curr_t, &end_t, &flags);
if (unlikely(!calculated_number_isnumber(value))) { if (unlikely(!netdata_double_isnumber(value))) {
// not collected // not collected
continue; continue;
} }
@ -156,7 +156,7 @@ calculated_number exporting_calculate_value_from_stored_data(
if (unlikely(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM)) if (unlikely(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM))
return sum; return sum;
return sum / (calculated_number)counter; return sum / (NETDATA_DOUBLE)counter;
} }
/** /**

18
exporting/prometheus/prometheus.c
View File

@ -362,7 +362,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
} }
} }
calculated_number value = rrdvar2number(rv); NETDATA_DOUBLE value = rrdvar2number(rv);
if (isnan(value) || isinf(value)) { if (isnan(value) || isinf(value)) {
if (opts->output_options & PROMETHEUS_OUTPUT_HELP) if (opts->output_options & PROMETHEUS_OUTPUT_HELP)
buffer_sprintf( buffer_sprintf(
@ -383,7 +383,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
buffer_sprintf( buffer_sprintf(
opts->wb, opts->wb,
"%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n", "%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
opts->prefix, opts->prefix,
opts->name, opts->name,
label_pre, label_pre,
@ -394,7 +394,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
else else
buffer_sprintf( buffer_sprintf(
opts->wb, opts->wb,
"%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n", "%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT "\n",
opts->prefix, opts->prefix,
opts->name, opts->name,
label_pre, label_pre,
@ -483,9 +483,9 @@ static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters
if (prometheus_collector) if (prometheus_collector)
buffer_sprintf( buffer_sprintf(
wb, wb,
CALCULATED_NUMBER_FORMAT, NETDATA_DOUBLE_FORMAT,
(calculated_number)p->rd->last_collected_value * (calculated_number)p->rd->multiplier / (NETDATA_DOUBLE)p->rd->last_collected_value * (NETDATA_DOUBLE)p->rd->multiplier /
(calculated_number)p->rd->divisor); (NETDATA_DOUBLE)p->rd->divisor);
else else
buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT, p->rd->last_collected_value); buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT, p->rd->last_collected_value);
@ -732,7 +732,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
time_t first_time = instance->after; time_t first_time = instance->after;
time_t last_time = instance->before; time_t last_time = instance->before;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_time); NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
if (!isnan(value) && !isinf(value)) { if (!isnan(value) && !isinf(value)) {
if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE) if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE)
@ -764,7 +764,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
buffer_sprintf( buffer_sprintf(
wb, wb,
"%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
" %llu\n", " %llu\n",
prefix, prefix,
context, context,
@ -779,7 +779,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
else else
buffer_sprintf( buffer_sprintf(
wb, wb,
"%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
"\n", "\n",
prefix, prefix,
context, context,

2
exporting/prometheus/remote_write/remote_write.c
View File

@ -294,7 +294,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
// we need average or sum of the data // we need average or sum of the data
time_t last_t = instance->before; time_t last_t = instance->before;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if (!isnan(value) && !isinf(value)) { if (!isnan(value) && !isinf(value)) {
if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE) if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE)

6
exporting/tests/exporting_doubles.c
View File

@ -52,11 +52,11 @@ int __wrap_mark_scheduled_instances(struct engine *engine)
return mock_type(int); return mock_type(int);
} }
calculated_number __real_exporting_calculate_value_from_stored_data( NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data(
struct instance *instance, struct instance *instance,
RRDDIM *rd, RRDDIM *rd,
time_t *last_timestamp); time_t *last_timestamp);
calculated_number __wrap_exporting_calculate_value_from_stored_data( NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data(
struct instance *instance, struct instance *instance,
RRDDIM *rd, RRDDIM *rd,
time_t *last_timestamp) time_t *last_timestamp)
@ -67,7 +67,7 @@ calculated_number __wrap_exporting_calculate_value_from_stored_data(
*last_timestamp = 15052; *last_timestamp = 15052;
function_called(); function_called();
return mock_type(calculated_number); return mock_type(NETDATA_DOUBLE);
} }
int __real_prepare_buffers(struct engine *engine); int __real_prepare_buffers(struct engine *engine);

6
exporting/tests/netdata_doubles.c
View File

@ -177,7 +177,7 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
return RRD_MEMORY_MODE_NONE_NAME; return RRD_MEMORY_MODE_NONE_NAME;
} }
calculated_number rrdvar2number(RRDVAR *rv) NETDATA_DOUBLE rrdvar2number(RRDVAR *rv)
{ {
(void)rv; (void)rv;
return 0; return 0;
@ -230,7 +230,7 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)
return mock_type(int); return mock_type(int);
} }
calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)
{ {
(void)handle; (void)handle;
(void)start_time; (void)start_time;
@ -239,7 +239,7 @@ calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *ha
function_called(); function_called();
return mock_type(calculated_number); return mock_type(NETDATA_DOUBLE);
} }
void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle) void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)

6
exporting/tests/test_exporting_engine.h
View File

@ -61,7 +61,7 @@ time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);
time_t __mock_rrddim_query_latest_time(RRDDIM *rd); time_t __mock_rrddim_query_latest_time(RRDDIM *rd);
void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time); void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle); int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);
calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags); NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle); void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
@ -88,11 +88,11 @@ int __wrap_init_connectors(struct engine *engine);
int __real_mark_scheduled_instances(struct engine *engine); int __real_mark_scheduled_instances(struct engine *engine);
int __wrap_mark_scheduled_instances(struct engine *engine); int __wrap_mark_scheduled_instances(struct engine *engine);
calculated_number __real_exporting_calculate_value_from_stored_data( NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data(
struct instance *instance, struct instance *instance,
RRDDIM *rd, RRDDIM *rd,
time_t *last_timestamp); time_t *last_timestamp);
calculated_number __wrap_exporting_calculate_value_from_stored_data( NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data(
struct instance *instance, struct instance *instance,
RRDDIM *rd, RRDDIM *rd,
time_t *last_timestamp); time_t *last_timestamp);

17
health/health.c
View File

@ -238,7 +238,7 @@ void health_reload(void) {
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// health main thread and friends // health main thread and friends
static inline RRDCALC_STATUS rrdcalc_value2status(calculated_number n) { static inline RRDCALC_STATUS rrdcalc_value2status(NETDATA_DOUBLE n) {
if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED; if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED;
if(n) return RRDCALC_STATUS_RAISED; if(n) return RRDCALC_STATUS_RAISED;
return RRDCALC_STATUS_CLEAR; return RRDCALC_STATUS_CLEAR;
@ -346,7 +346,9 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN"); char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN");
snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" CALCULATED_NUMBER_FORMAT_ZERO "' '" CALCULATED_NUMBER_FORMAT_ZERO "' '%s' '%u' '%u' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d' '%s' '%s' '%s' '%s' '%s'", snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" NETDATA_DOUBLE_FORMAT_ZERO
"' '" NETDATA_DOUBLE_FORMAT_ZERO
"' '%s' '%u' '%u' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d' '%s' '%s' '%s' '%s' '%s'",
exec, exec,
recipient, recipient,
host->registry_hostname, host->registry_hostname,
@ -411,7 +413,7 @@ static inline void health_alarm_wait_for_execution(ALARM_ENTRY *ae) {
} }
static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) { static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
debug(D_HEALTH, "Health alarm '%s.%s' = " CALCULATED_NUMBER_FORMAT_AUTO " - changed status from %s to %s", debug(D_HEALTH, "Health alarm '%s.%s' = " NETDATA_DOUBLE_FORMAT_AUTO " - changed status from %s to %s",
ae->chart?ae->chart:"NOCHART", ae->name, ae->chart?ae->chart:"NOCHART", ae->name,
ae->new_value, ae->new_value,
rrdcalc_status2string(ae->old_status), rrdcalc_status2string(ae->old_status),
@ -892,8 +894,7 @@ void *health_main(void *ptr) {
} else } else
rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN; rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT, host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
CALCULATED_NUMBER_FORMAT, host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
rc->value rc->value
); );
} }
@ -917,7 +918,7 @@ void *health_main(void *ptr) {
rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR; rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value " debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
CALCULATED_NUMBER_FORMAT NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
rc->calculation->parsed_as, rc->calculation->result, rc->calculation->parsed_as, rc->calculation->result,
buffer_tostring(rc->calculation->error_msg), rc->source buffer_tostring(rc->calculation->error_msg), rc->source
@ -966,7 +967,7 @@ void *health_main(void *ptr) {
} else { } else {
rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR; rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value " debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
CALCULATED_NUMBER_FORMAT NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART",
rc->name, rc->warning->result, buffer_tostring(rc->warning->error_msg), rc->source rc->name, rc->warning->result, buffer_tostring(rc->warning->error_msg), rc->source
); );
@ -992,7 +993,7 @@ void *health_main(void *ptr) {
} else { } else {
rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR; rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value " debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
CALCULATED_NUMBER_FORMAT NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART",
rc->name, rc->critical->result, buffer_tostring(rc->critical->error_msg), rc->name, rc->critical->result, buffer_tostring(rc->critical->error_msg),
rc->source rc->source

6
health/health.h
View File

@ -35,7 +35,7 @@ extern void health_init(void);
extern void health_reload(void); extern void health_reload(void);
extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result); extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, NETDATA_DOUBLE *result);
extern void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* context, RRDCALC_STATUS status); extern void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* context, RRDCALC_STATUS status);
extern void health_alarms2json(RRDHOST *host, BUFFER *wb, int all); extern void health_alarms2json(RRDHOST *host, BUFFER *wb, int all);
extern void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all); extern void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all);
@ -63,8 +63,8 @@ extern ALARM_ENTRY* health_create_alarm_entry(
const char *exec, const char *exec,
const char *recipient, const char *recipient,
time_t duration, time_t duration,
calculated_number old_value, NETDATA_DOUBLE old_value,
calculated_number new_value, NETDATA_DOUBLE new_value,
RRDCALC_STATUS old_status, RRDCALC_STATUS old_status,
RRDCALC_STATUS new_status, RRDCALC_STATUS new_status,
const char *source, const char *source,

16
health/health_config.c
View File

@ -54,7 +54,9 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id); rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id);
debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO
", red " NETDATA_DOUBLE_FORMAT_AUTO
", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
rc->chart?rc->chart:"NOCHART", rc->chart?rc->chart:"NOCHART",
rc->name, rc->name,
rc->id, rc->id,
@ -141,7 +143,9 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL
} }
} }
debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u", debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO
", red " NETDATA_DOUBLE_FORMAT_AUTO
", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
rt->name, rt->name,
(rt->context)?rt->context:"NONE", (rt->context)?rt->context:"NONE",
(rt->exec)?rt->exec:"DEFAULT", (rt->exec)?rt->exec:"DEFAULT",
@ -848,7 +852,7 @@ static int health_readfile(const char *filename, void *data) {
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) { else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
alert_cfg->green = strdupz(value); alert_cfg->green = strdupz(value);
char *e; char *e;
rc->green = str2ld(value, &e); rc->green = str2ndd(value, &e);
if(e && *e) { if(e && *e) {
error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
line, filename, rc->name, key, e); line, filename, rc->name, key, e);
@ -857,7 +861,7 @@ static int health_readfile(const char *filename, void *data) {
else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) { else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
alert_cfg->red = strdupz(value); alert_cfg->red = strdupz(value);
char *e; char *e;
rc->red = str2ld(value, &e); rc->red = str2ndd(value, &e);
if(e && *e) { if(e && *e) {
error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
line, filename, rc->name, key, e); line, filename, rc->name, key, e);
@ -1097,7 +1101,7 @@ static int health_readfile(const char *filename, void *data) {
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) { else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
alert_cfg->green = strdupz(value); alert_cfg->green = strdupz(value);
char *e; char *e;
rt->green = str2ld(value, &e); rt->green = str2ndd(value, &e);
if(e && *e) { if(e && *e) {
error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
line, filename, rt->name, key, e); line, filename, rt->name, key, e);
@ -1106,7 +1110,7 @@ static int health_readfile(const char *filename, void *data) {
else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) { else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
alert_cfg->red = strdupz(value); alert_cfg->red = strdupz(value);
char *e; char *e;
rt->red = str2ld(value, &e); rt->red = str2ndd(value, &e);
if(e && *e) { if(e && *e) {
error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
line, filename, rt->name, key, e); line, filename, rt->name, key, e);

6
health/health_log.c
View File

@ -98,7 +98,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
"\t%08x\t%08x\t%08x" "\t%08x\t%08x\t%08x"
"\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"
"\t%d\t%d\t%d\t%d" "\t%d\t%d\t%d\t%d"
"\t" CALCULATED_NUMBER_FORMAT_AUTO "\t" CALCULATED_NUMBER_FORMAT_AUTO "\t" NETDATA_DOUBLE_FORMAT_AUTO "\t" NETDATA_DOUBLE_FORMAT_AUTO
"\t%016"PRIx64"" "\t%016"PRIx64""
"\t%s\t%s\t%s" "\t%s\t%s\t%s"
"\n" "\n"
@ -457,8 +457,8 @@ inline ALARM_ENTRY* health_create_alarm_entry(
const char *exec, const char *exec,
const char *recipient, const char *recipient,
time_t duration, time_t duration,
calculated_number old_value, NETDATA_DOUBLE old_value,
calculated_number new_value, NETDATA_DOUBLE new_value,
RRDCALC_STATUS old_status, RRDCALC_STATUS old_status,
RRDCALC_STATUS new_status, RRDCALC_STATUS new_status,
const char *source, const char *source,

8
libnetdata/buffer/buffer.c
View File

@ -65,9 +65,9 @@ void buffer_char_replace(BUFFER *wb, char from, char to)
} }
// This trick seems to give an 80% speed increase in 32bit systems // This trick seems to give an 80% speed increase in 32bit systems
// print_calculated_number_llu_r() will just print the digits up to the // print_number_llu_r() will just print the digits up to the
// point the remaining value fits in 32 bits, and then calls // point the remaining value fits in 32 bits, and then calls
// print_calculated_number_lu_r() to print the rest with 32 bit arithmetic. // print_number_lu_r() to print the rest with 32 bit arithmetic.
inline char *print_number_lu_r(char *str, unsigned long uvalue) { inline char *print_number_lu_r(char *str, unsigned long uvalue) {
char *wstr = str; char *wstr = str;
@ -299,7 +299,7 @@ void buffer_sprintf(BUFFER *wb, const char *fmt, ...)
} }
void buffer_rrd_value(BUFFER *wb, calculated_number value) void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value)
{ {
buffer_need_bytes(wb, 50); buffer_need_bytes(wb, 50);
@ -308,7 +308,7 @@ void buffer_rrd_value(BUFFER *wb, calculated_number value)
return; return;
} }
else else
wb->len += print_calculated_number(&wb->buffer[wb->len], value); wb->len += print_netdata_double(&wb->buffer[wb->len], value);
// terminate it // terminate it
buffer_need_bytes(wb, 1); buffer_need_bytes(wb, 1);

2
libnetdata/buffer/buffer.h
View File

@ -56,7 +56,7 @@ extern void buffer_reset(BUFFER *wb);
extern void buffer_strcat(BUFFER *wb, const char *txt); extern void buffer_strcat(BUFFER *wb, const char *txt);
extern void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len); extern void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len);
extern void buffer_rrd_value(BUFFER *wb, calculated_number value); extern void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value);
extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
extern void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); extern void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);

2
libnetdata/clocks/clocks.c
View File

@ -393,7 +393,7 @@ static inline collected_number read_proc_uptime(char *filename) {
return 0; return 0;
} }
return (collected_number)(strtold(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0); return (collected_number)(strtondd(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0);
} }
inline collected_number uptime_msec(char *filename){ inline collected_number uptime_msec(char *filename){

47
libnetdata/config/appconfig.c
View File

@ -462,15 +462,15 @@ long long appconfig_get_number(struct config *root, const char *section, const c
return strtoll(s, NULL, 0); return strtoll(s, NULL, 0);
} }
LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value) NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value)
{ {
char buffer[100], *s; char buffer[100], *s;
sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value); sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value);
s = appconfig_get(root, section, name, buffer); s = appconfig_get(root, section, name, buffer);
if(!s) return value; if(!s) return value;
return str2ld(s, NULL); return str2ndd(s, NULL);
} }
static inline int appconfig_test_boolean_value(char *s) { static inline int appconfig_test_boolean_value(char *s) {
@ -588,10 +588,10 @@ long long appconfig_set_number(struct config *root, const char *section, const c
return value; return value;
} }
LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value) NETDATA_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value)
{ {
char buffer[100]; char buffer[100];
sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value); sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value);
appconfig_set(root, section, name, buffer); appconfig_set(root, section, name, buffer);
@ -831,26 +831,27 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
"#\n" "#\n"
"\n# global netdata configuration\n"); "\n# global netdata configuration\n");
for(i = 0; i <= 15 ;i++) { for(i = 0; i <= 16 ;i++) {
appconfig_wrlock(root); appconfig_wrlock(root);
for(co = root->first_section; co ; co = co->next) { for(co = root->first_section; co ; co = co->next) {
if(!strcmp(co->name, CONFIG_SECTION_GLOBAL)) pri = 0; if(!strcmp(co->name, CONFIG_SECTION_GLOBAL)) pri = 0;
else if(!strcmp(co->name, CONFIG_SECTION_DIRECTORIES)) pri = 1; else if(!strcmp(co->name, CONFIG_SECTION_DB)) pri = 1;
else if(!strcmp(co->name, CONFIG_SECTION_LOGS)) pri = 2; else if(!strcmp(co->name, CONFIG_SECTION_DIRECTORIES)) pri = 2;
else if(!strcmp(co->name, CONFIG_SECTION_ENV_VARS)) pri = 3; else if(!strcmp(co->name, CONFIG_SECTION_LOGS)) pri = 3;
else if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) pri = 4; else if(!strcmp(co->name, CONFIG_SECTION_ENV_VARS)) pri = 4;
else if(!strcmp(co->name, CONFIG_SECTION_SQLITE)) pri = 5; else if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) pri = 5;
else if(!strcmp(co->name, CONFIG_SECTION_CLOUD)) pri = 6; else if(!strcmp(co->name, CONFIG_SECTION_SQLITE)) pri = 6;
else if(!strcmp(co->name, CONFIG_SECTION_ML)) pri = 7; else if(!strcmp(co->name, CONFIG_SECTION_CLOUD)) pri = 7;
else if(!strcmp(co->name, CONFIG_SECTION_HEALTH)) pri = 8; else if(!strcmp(co->name, CONFIG_SECTION_ML)) pri = 8;
else if(!strcmp(co->name, CONFIG_SECTION_WEB)) pri = 9; else if(!strcmp(co->name, CONFIG_SECTION_HEALTH)) pri = 9;
// by default, new sections will get pri = 10 (set at the end, below) else if(!strcmp(co->name, CONFIG_SECTION_WEB)) pri = 10;
else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 11; // by default, new sections will get pri = 11 (set at the end, below)
else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 12; else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 12;
else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 13; else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 13;
else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 14; else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 14;
else if(!strncmp(co->name, "plugin:", 7)) pri = 15; // << change the loop too if you change this else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 15;
else pri = 10; // this is used for any new (currently unknown) sections else if(!strncmp(co->name, "plugin:", 7)) pri = 16; // << change the loop too if you change this
else pri = 11; // this is used for any new (currently unknown) sections
if(i == pri) { if(i == pri) {
int loaded = 0; int loaded = 0;
@ -916,7 +917,7 @@ int config_parse_duration(const char* string, int* result) {
if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback; if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback;
char *e = NULL; char *e = NULL;
calculated_number n = str2ld(string, &e); NETDATA_DOUBLE n = str2ndd(string, &e);
if(e && *e) { if(e && *e) {
switch (*e) { switch (*e) {
case 'Y': case 'Y':

6
libnetdata/config/appconfig.h
View File

@ -100,6 +100,8 @@
#define CONFIG_SECTION_HOST_LABEL "host labels" #define CONFIG_SECTION_HOST_LABEL "host labels"
#define EXPORTING_CONF "exporting.conf" #define EXPORTING_CONF "exporting.conf"
#define CONFIG_SECTION_GLOBAL_STATISTICS "global statistics" #define CONFIG_SECTION_GLOBAL_STATISTICS "global statistics"
#define CONFIG_SECTION_DB "db"
// these are used to limit the configuration names and values lengths // these are used to limit the configuration names and values lengths
// they are not enforced by config.c functions (they will strdup() all strings, no matter of their length) // they are not enforced by config.c functions (they will strdup() all strings, no matter of their length)
@ -168,7 +170,7 @@ extern void config_section_unlock(struct section *co);
extern char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value); extern char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value);
extern char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value); extern char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value);
extern long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value); extern long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value);
extern LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value); extern NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value);
extern int appconfig_get_boolean_by_section(struct section *co, const char *name, int value); extern int appconfig_get_boolean_by_section(struct section *co, const char *name, int value);
extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value); extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value);
extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value); extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value);
@ -177,7 +179,7 @@ extern int appconfig_get_duration(struct config *root, const char *section, cons
extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value); extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value);
extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value); extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value);
extern long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value); extern long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value);
extern LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value); extern NETDATA_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value);
extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value); extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value);
extern int appconfig_exists(struct config *root, const char *section, const char *name); extern int appconfig_exists(struct config *root, const char *section, const char *name);

112
libnetdata/eval/eval.c
View File

@ -9,7 +9,7 @@ typedef struct eval_value {
int type; int type;
union { union {
calculated_number number; NETDATA_DOUBLE number;
EVAL_VARIABLE *variable; EVAL_VARIABLE *variable;
struct eval_node *expression; struct eval_node *expression;
}; };
@ -54,16 +54,16 @@ typedef struct eval_node {
static inline void eval_node_free(EVAL_NODE *op); static inline void eval_node_free(EVAL_NODE *op);
static inline EVAL_NODE *parse_full_expression(const char **string, int *error); static inline EVAL_NODE *parse_full_expression(const char **string, int *error);
static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error); static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error);
static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error); static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error); static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error);
static inline void print_parsed_as_constant(BUFFER *out, calculated_number n); static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n);
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// evaluation of expressions // evaluation of expressions
static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) { static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) {
static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0; static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0;
calculated_number n; NETDATA_DOUBLE n;
if(unlikely(this_hash == 0)) { if(unlikely(this_hash == 0)) {
this_hash = simple_hash("this"); this_hash = simple_hash("this");
@ -179,8 +179,8 @@ static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABL
return NAN; return NAN;
} }
static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) { static inline NETDATA_DOUBLE eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) {
calculated_number n; NETDATA_DOUBLE n;
switch(v->type) { switch(v->type) {
case EVAL_VALUE_EXPRESSION: case EVAL_VALUE_EXPRESSION:
@ -204,101 +204,101 @@ static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v,
return n; return n;
} }
static inline int is_true(calculated_number n) { static inline int is_true(NETDATA_DOUBLE n) {
if(isnan(n)) return 0; if(isnan(n)) return 0;
if(isinf(n)) return 1; if(isinf(n)) return 1;
if(n == 0) return 0; if(n == 0) return 0;
return 1; return 1;
} }
calculated_number eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return is_true(eval_value(exp, &op->ops[0], error)) && is_true(eval_value(exp, &op->ops[1], error)); return is_true(eval_value(exp, &op->ops[0], error)) && is_true(eval_value(exp, &op->ops[1], error));
} }
calculated_number eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return is_true(eval_value(exp, &op->ops[0], error)) || is_true(eval_value(exp, &op->ops[1], error)); return is_true(eval_value(exp, &op->ops[0], error)) || is_true(eval_value(exp, &op->ops[1], error));
} }
calculated_number eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
return isgreaterequal(n1, n2); return isgreaterequal(n1, n2);
} }
calculated_number eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
return islessequal(n1, n2); return islessequal(n1, n2);
} }
calculated_number eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) && isnan(n2)) return 1; if(isnan(n1) && isnan(n2)) return 1;
if(isinf(n1) && isinf(n2)) return 1; if(isinf(n1) && isinf(n2)) return 1;
if(isnan(n1) || isnan(n2)) return 0; if(isnan(n1) || isnan(n2)) return 0;
if(isinf(n1) || isinf(n2)) return 0; if(isinf(n1) || isinf(n2)) return 0;
return calculated_number_equal(n1, n2); return considered_equal_ndd(n1, n2);
} }
calculated_number eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return !eval_equal(exp, op, error); return !eval_equal(exp, op, error);
} }
calculated_number eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
return isless(n1, n2); return isless(n1, n2);
} }
calculated_number eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
return isgreater(n1, n2); return isgreater(n1, n2);
} }
calculated_number eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) || isnan(n2)) return NAN; if(isnan(n1) || isnan(n2)) return NAN;
if(isinf(n1) || isinf(n2)) return INFINITY; if(isinf(n1) || isinf(n2)) return INFINITY;
return n1 + n2; return n1 + n2;
} }
calculated_number eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) || isnan(n2)) return NAN; if(isnan(n1) || isnan(n2)) return NAN;
if(isinf(n1) || isinf(n2)) return INFINITY; if(isinf(n1) || isinf(n2)) return INFINITY;
return n1 - n2; return n1 - n2;
} }
calculated_number eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) || isnan(n2)) return NAN; if(isnan(n1) || isnan(n2)) return NAN;
if(isinf(n1) || isinf(n2)) return INFINITY; if(isinf(n1) || isinf(n2)) return INFINITY;
return n1 * n2; return n1 * n2;
} }
calculated_number eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error); NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) || isnan(n2)) return NAN; if(isnan(n1) || isnan(n2)) return NAN;
if(isinf(n1) || isinf(n2)) return INFINITY; if(isinf(n1) || isinf(n2)) return INFINITY;
return n1 / n2; return n1 / n2;
} }
calculated_number eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return eval_value(exp, &op->ops[0], error); return eval_value(exp, &op->ops[0], error);
} }
calculated_number eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return !is_true(eval_value(exp, &op->ops[0], error)); return !is_true(eval_value(exp, &op->ops[0], error));
} }
calculated_number eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return eval_value(exp, &op->ops[0], error); return eval_value(exp, &op->ops[0], error);
} }
calculated_number eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
if(isnan(n1)) return NAN; if(isnan(n1)) return NAN;
if(isinf(n1)) return INFINITY; if(isinf(n1)) return INFINITY;
return -n1; return -n1;
} }
calculated_number eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error); NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
if(isnan(n1)) return NAN; if(isnan(n1)) return NAN;
if(isinf(n1)) return INFINITY; if(isinf(n1)) return INFINITY;
return ABS(n1); return ABS(n1);
} }
calculated_number eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { NETDATA_DOUBLE eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
if(is_true(eval_value(exp, &op->ops[0], error))) if(is_true(eval_value(exp, &op->ops[0], error)))
return eval_value(exp, &op->ops[1], error); return eval_value(exp, &op->ops[1], error);
else else
@ -310,7 +310,7 @@ static struct operator {
char precedence; char precedence;
char parameters; char parameters;
char isfunction; char isfunction;
calculated_number (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error); NETDATA_DOUBLE (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
} operators[256] = { } operators[256] = {
// this is a random access array // this is a random access array
// we always access it with a known EVAL_OPERATOR_X // we always access it with a known EVAL_OPERATOR_X
@ -341,13 +341,13 @@ static struct operator {
#define eval_precedence(operator) (operators[(unsigned char)(operator)].precedence) #define eval_precedence(operator) (operators[(unsigned char)(operator)].precedence)
static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
if(unlikely(op->count != operators[op->operator].parameters)) { if(unlikely(op->count != operators[op->operator].parameters)) {
*error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS; *error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS;
return 0; return 0;
} }
calculated_number n = operators[op->operator].eval(exp, op, error); NETDATA_DOUBLE n = operators[op->operator].eval(exp, op, error);
return n; return n;
} }
@ -360,7 +360,7 @@ static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *
buffer_sprintf(out, "${%s}", v->name); buffer_sprintf(out, "${%s}", v->name);
} }
static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) { static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n) {
if(unlikely(isnan(n))) { if(unlikely(isnan(n))) {
buffer_strcat(out, "nan"); buffer_strcat(out, "nan");
return; return;
@ -372,7 +372,7 @@ static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) {
} }
char b[100+1], *s; char b[100+1], *s;
snprintfz(b, 100, CALCULATED_NUMBER_FORMAT, n); snprintfz(b, 100, NETDATA_DOUBLE_FORMAT, n);
s = &b[strlen(b) - 1]; s = &b[strlen(b) - 1];
while(s > b && *s == '0') { while(s > b && *s == '0') {
@ -737,9 +737,9 @@ static inline int parse_variable(const char **string, char *buffer, size_t len)
return 0; return 0;
} }
static inline int parse_constant(const char **string, calculated_number *number) { static inline int parse_constant(const char **string, NETDATA_DOUBLE *number) {
char *end = NULL; char *end = NULL;
calculated_number n = str2ld(*string, &end); NETDATA_DOUBLE n = str2ndd(*string, &end);
if(unlikely(!end || *string == end)) { if(unlikely(!end || *string == end)) {
*number = 0; *number = 0;
return 0; return 0;
@ -845,7 +845,7 @@ static inline void eval_node_set_value_to_node(EVAL_NODE *op, int pos, EVAL_NODE
op->ops[pos].expression = value; op->ops[pos].expression = value;
} }
static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, calculated_number value) { static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, NETDATA_DOUBLE value) {
if(pos >= op->count) if(pos >= op->count)
fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1); fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
@ -911,7 +911,7 @@ static inline EVAL_NODE *parse_next_operand_given_its_operator(const char **stri
static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error) { static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error) {
char variable_buffer[EVAL_MAX_VARIABLE_NAME_LENGTH + 1]; char variable_buffer[EVAL_MAX_VARIABLE_NAME_LENGTH + 1];
EVAL_NODE *op1 = NULL; EVAL_NODE *op1 = NULL;
calculated_number number; NETDATA_DOUBLE number;
*error = EVAL_ERROR_OK; *error = EVAL_ERROR_OK;

6
libnetdata/eval/eval.h
View File

@ -28,11 +28,11 @@ typedef struct eval_expression {
const char *parsed_as; const char *parsed_as;
RRDCALC_STATUS *status; RRDCALC_STATUS *status;
calculated_number *myself; NETDATA_DOUBLE *myself;
time_t *after; time_t *after;
time_t *before; time_t *before;
calculated_number result; NETDATA_DOUBLE result;
int error; int error;
BUFFER *error_msg; BUFFER *error_msg;
@ -83,6 +83,6 @@ extern const char *expression_strerror(int error);
// 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg) // 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg)
extern int expression_evaluate(EVAL_EXPRESSION *expression); extern int expression_evaluate(EVAL_EXPRESSION *expression);
extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result); extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result);
#endif //NETDATA_EVAL_H #endif //NETDATA_EVAL_H

71
libnetdata/inlined.h
View File

@ -151,77 +151,6 @@ static inline long long str2ll(const char *s, char **endptr) {
return n; return n;
} }
static inline long double str2ld(const char *s, char **endptr) {
int negative = 0;
const char *start = s;
unsigned long long integer_part = 0;
unsigned long decimal_part = 0;
size_t decimal_digits = 0;
switch(*s) {
case '-':
s++;
negative = 1;
break;
case '+':
s++;
break;
case 'n':
if(s[1] == 'a' && s[2] == 'n') {
if(endptr) *endptr = (char *)&s[3];
return NAN;
}
break;
case 'i':
if(s[1] == 'n' && s[2] == 'f') {
if(endptr) *endptr = (char *)&s[3];
return INFINITY;
}
break;
default:
break;
}
while (*s >= '0' && *s <= '9') {
integer_part = (integer_part * 10) + (*s - '0');
s++;
}
if(unlikely(*s == '.')) {
decimal_part = 0;
s++;
while (*s >= '0' && *s <= '9') {
decimal_part = (decimal_part * 10) + (*s - '0');
s++;
decimal_digits++;
}
}
if(unlikely(*s == 'e' || *s == 'E'))
return strtold(start, endptr);
if(unlikely(endptr))
*endptr = (char *)s;
if(unlikely(negative)) {
if(unlikely(decimal_digits))
return -((long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits));
else
return -((long double)integer_part);
}
else {
if(unlikely(decimal_digits))
return (long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits);
else
return (long double)integer_part;
}
}
static inline char *strncpyz(char *dst, const char *src, size_t n) { static inline char *strncpyz(char *dst, const char *src, size_t n) {
char *p = dst; char *p = dst;

4
libnetdata/json/json.c
View File

@ -111,7 +111,7 @@ int json_callback_print(JSON_ENTRY *e)
break; break;
case JSON_NUMBER: case JSON_NUMBER:
sprintf(txt,"%Lf", e->data.number); sprintf(txt, NETDATA_DOUBLE_FORMAT_AUTO, e->data.number);
buffer_strcat(wb,txt); buffer_strcat(wb,txt);
break; break;
@ -168,7 +168,7 @@ static inline void json_jsonc_set_integer(JSON_ENTRY *e, char *key, int64_t valu
e->type = JSON_NUMBER; e->type = JSON_NUMBER;
memcpy(e->name, key, len); memcpy(e->name, key, len);
e->name[len] = 0; e->name[len] = 0;
e->data.number = value; e->data.number = (NETDATA_DOUBLE)value;
} }
/** /**

10
libnetdata/json/json.h
View File

@ -31,12 +31,12 @@ typedef struct json_entry {
char name[JSON_NAME_LEN + 1]; char name[JSON_NAME_LEN + 1];
char fullname[JSON_FULLNAME_LEN + 1]; char fullname[JSON_FULLNAME_LEN + 1];
union { union {
char *string; // type == JSON_STRING char *string; // type == JSON_STRING
long double number; // type == JSON_NUMBER NETDATA_DOUBLE number; // type == JSON_NUMBER
int boolean; // type == JSON_BOOLEAN int boolean; // type == JSON_BOOLEAN
size_t items; // type == JSON_ARRAY size_t items; // type == JSON_ARRAY
} data; } data;
size_t pos; // the position of this item in its parent size_t pos; // the position of this item in its parent
char *original_string; char *original_string;

2
libnetdata/required_dummies.h
View File

@ -23,7 +23,7 @@ void signals_unblock(void){};
void signals_reset(void){}; void signals_reset(void){};
// callback required by eval() // callback required by eval()
int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result) int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result)
{ {
(void)variable; (void)variable;
(void)hash; (void)hash;

180
libnetdata/statistical/statistical.c
View File

@ -2,28 +2,28 @@
#include "../libnetdata.h" #include "../libnetdata.h"
LONG_DOUBLE default_single_exponential_smoothing_alpha = 0.1; NETDATA_DOUBLE default_single_exponential_smoothing_alpha = 0.1;
void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg) { void log_series_to_stderr(NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE result, const char *msg) {
const LONG_DOUBLE *value, *end = &series[entries]; const NETDATA_DOUBLE *value, *end = &series[entries];
fprintf(stderr, "%s of %zu entries [ ", msg, entries); fprintf(stderr, "%s of %zu entries [ ", msg, entries);
for(value = series; value < end ;value++) { for(value = series; value < end ;value++) {
if(value != series) fprintf(stderr, ", "); if(value != series) fprintf(stderr, ", ");
fprintf(stderr, "%" LONG_DOUBLE_MODIFIER, *value); fprintf(stderr, "%" NETDATA_DOUBLE_MODIFIER, *value);
} }
fprintf(stderr, " ] results in " CALCULATED_NUMBER_FORMAT "\n", result); fprintf(stderr, " ] results in " NETDATA_DOUBLE_FORMAT "\n", result);
} }
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
inline LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count) { inline NETDATA_DOUBLE sum_and_count(const NETDATA_DOUBLE *series, size_t entries, size_t *count) {
const LONG_DOUBLE *value, *end = &series[entries]; const NETDATA_DOUBLE *value, *end = &series[entries];
LONG_DOUBLE sum = 0; NETDATA_DOUBLE sum = 0;
size_t c = 0; size_t c = 0;
for(value = series; value < end ; value++) { for(value = series; value < end ; value++) {
if(calculated_number_isnumber(*value)) { if(netdata_double_isnumber(*value)) {
sum += *value; sum += *value;
c++; c++;
} }
@ -35,42 +35,42 @@ inline LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size
return sum; return sum;
} }
inline LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries) { inline NETDATA_DOUBLE sum(const NETDATA_DOUBLE *series, size_t entries) {
return sum_and_count(series, entries, NULL); return sum_and_count(series, entries, NULL);
} }
inline LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries) { inline NETDATA_DOUBLE average(const NETDATA_DOUBLE *series, size_t entries) {
size_t count = 0; size_t count = 0;
LONG_DOUBLE sum = sum_and_count(series, entries, &count); NETDATA_DOUBLE sum = sum_and_count(series, entries, &count);
if(unlikely(!count)) return NAN; if(unlikely(!count)) return NAN;
return sum / (LONG_DOUBLE)count; return sum / (NETDATA_DOUBLE)count;
} }
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period) { NETDATA_DOUBLE moving_average(const NETDATA_DOUBLE *series, size_t entries, size_t period) {
if(unlikely(period <= 0)) if(unlikely(period <= 0))
return 0.0; return 0.0;
size_t i, count; size_t i, count;
LONG_DOUBLE sum = 0, avg = 0; NETDATA_DOUBLE sum = 0, avg = 0;
LONG_DOUBLE p[period]; NETDATA_DOUBLE p[period];
for(count = 0; count < period ; count++) for(count = 0; count < period ; count++)
p[count] = 0.0; p[count] = 0.0;
for(i = 0, count = 0; i < entries; i++) { for(i = 0, count = 0; i < entries; i++) {
LONG_DOUBLE value = series[i]; NETDATA_DOUBLE value = series[i];
if(unlikely(!calculated_number_isnumber(value))) continue; if(unlikely(!netdata_double_isnumber(value))) continue;
if(unlikely(count < period)) { if(unlikely(count < period)) {
sum += value; sum += value;
avg = (count == period - 1) ? sum / (LONG_DOUBLE)period : 0; avg = (count == period - 1) ? sum / (NETDATA_DOUBLE)period : 0;
} }
else { else {
sum = sum - p[count % period] + value; sum = sum - p[count % period] + value;
avg = sum / (LONG_DOUBLE)period; avg = sum / (NETDATA_DOUBLE)period;
} }
p[count % period] = value; p[count % period] = value;
@ -83,8 +83,8 @@ LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t per
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
static int qsort_compare(const void *a, const void *b) { static int qsort_compare(const void *a, const void *b) {
LONG_DOUBLE *p1 = (LONG_DOUBLE *)a, *p2 = (LONG_DOUBLE *)b; NETDATA_DOUBLE *p1 = (NETDATA_DOUBLE *)a, *p2 = (NETDATA_DOUBLE *)b;
LONG_DOUBLE n1 = *p1, n2 = *p2; NETDATA_DOUBLE n1 = *p1, n2 = *p2;
if(unlikely(isnan(n1) || isnan(n2))) { if(unlikely(isnan(n1) || isnan(n2))) {
if(isnan(n1) && !isnan(n2)) return -1; if(isnan(n1) && !isnan(n2)) return -1;
@ -102,22 +102,22 @@ static int qsort_compare(const void *a, const void *b) {
return 0; return 0;
} }
inline void sort_series(LONG_DOUBLE *series, size_t entries) { inline void sort_series(NETDATA_DOUBLE *series, size_t entries) {
qsort(series, entries, sizeof(LONG_DOUBLE), qsort_compare); qsort(series, entries, sizeof(NETDATA_DOUBLE), qsort_compare);
} }
inline LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries) { inline NETDATA_DOUBLE *copy_series(const NETDATA_DOUBLE *series, size_t entries) {
LONG_DOUBLE *copy = mallocz(sizeof(LONG_DOUBLE) * entries); NETDATA_DOUBLE *copy = mallocz(sizeof(NETDATA_DOUBLE) * entries);
memcpy(copy, series, sizeof(LONG_DOUBLE) * entries); memcpy(copy, series, sizeof(NETDATA_DOUBLE) * entries);
return copy; return copy;
} }
LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries) { NETDATA_DOUBLE median_on_sorted_series(const NETDATA_DOUBLE *series, size_t entries) {
if(unlikely(entries == 0)) return NAN; if(unlikely(entries == 0)) return NAN;
if(unlikely(entries == 1)) return series[0]; if(unlikely(entries == 1)) return series[0];
if(unlikely(entries == 2)) return (series[0] + series[1]) / 2; if(unlikely(entries == 2)) return (series[0] + series[1]) / 2;
LONG_DOUBLE average; NETDATA_DOUBLE average;
if(entries % 2 == 0) { if(entries % 2 == 0) {
size_t m = entries / 2; size_t m = entries / 2;
average = (series[m] + series[m + 1]) / 2; average = (series[m] + series[m + 1]) / 2;
@ -129,17 +129,17 @@ LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries) {
return average; return average;
} }
LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries) { NETDATA_DOUBLE median(const NETDATA_DOUBLE *series, size_t entries) {
if(unlikely(entries == 0)) return NAN; if(unlikely(entries == 0)) return NAN;
if(unlikely(entries == 1)) return series[0]; if(unlikely(entries == 1)) return series[0];
if(unlikely(entries == 2)) if(unlikely(entries == 2))
return (series[0] + series[1]) / 2; return (series[0] + series[1]) / 2;
LONG_DOUBLE *copy = copy_series(series, entries); NETDATA_DOUBLE *copy = copy_series(series, entries);
sort_series(copy, entries); sort_series(copy, entries);
LONG_DOUBLE avg = median_on_sorted_series(copy, entries); NETDATA_DOUBLE avg = median_on_sorted_series(copy, entries);
freez(copy); freez(copy);
return avg; return avg;
@ -147,18 +147,18 @@ LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries) {
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period) { NETDATA_DOUBLE moving_median(const NETDATA_DOUBLE *series, size_t entries, size_t period) {
if(entries <= period) if(entries <= period)
return median(series, entries); return median(series, entries);
LONG_DOUBLE *data = copy_series(series, entries); NETDATA_DOUBLE *data = copy_series(series, entries);
size_t i; size_t i;
for(i = period; i < entries; i++) { for(i = period; i < entries; i++) {
data[i - period] = median(&series[i - period], period); data[i - period] = median(&series[i - period], period);
} }
LONG_DOUBLE avg = median(data, entries - period); NETDATA_DOUBLE avg = median(data, entries - period);
freez(data); freez(data);
return avg; return avg;
} }
@ -166,17 +166,17 @@ LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t peri
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
// http://stackoverflow.com/a/15150143/4525767 // http://stackoverflow.com/a/15150143/4525767
LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries) { NETDATA_DOUBLE running_median_estimate(const NETDATA_DOUBLE *series, size_t entries) {
LONG_DOUBLE median = 0.0f; NETDATA_DOUBLE median = 0.0f;
LONG_DOUBLE average = 0.0f; NETDATA_DOUBLE average = 0.0f;
size_t i; size_t i;
for(i = 0; i < entries ; i++) { for(i = 0; i < entries ; i++) {
LONG_DOUBLE value = series[i]; NETDATA_DOUBLE value = series[i];
if(unlikely(!calculated_number_isnumber(value))) continue; if(unlikely(!netdata_double_isnumber(value))) continue;
average += ( value - average ) * 0.1f; // rough running average. average += ( value - average ) * 0.1f; // rough running average.
median += copysignl( average * 0.01, value - median ); median += copysignndd( average * 0.01, value - median );
} }
return median; return median;
@ -184,16 +184,16 @@ LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries) {
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries) { NETDATA_DOUBLE standard_deviation(const NETDATA_DOUBLE *series, size_t entries) {
if(unlikely(entries == 0)) return NAN; if(unlikely(entries == 0)) return NAN;
if(unlikely(entries == 1)) return series[0]; if(unlikely(entries == 1)) return series[0];
const LONG_DOUBLE *value, *end = &series[entries]; const NETDATA_DOUBLE *value, *end = &series[entries];
size_t count; size_t count;
LONG_DOUBLE sum; NETDATA_DOUBLE sum;
for(count = 0, sum = 0, value = series ; value < end ;value++) { for(count = 0, sum = 0, value = series ; value < end ;value++) {
if(likely(calculated_number_isnumber(*value))) { if(likely(netdata_double_isnumber(*value))) {
count++; count++;
sum += *value; sum += *value;
} }
@ -202,55 +202,55 @@ LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries) {
if(unlikely(count == 0)) return NAN; if(unlikely(count == 0)) return NAN;
if(unlikely(count == 1)) return sum; if(unlikely(count == 1)) return sum;
LONG_DOUBLE average = sum / (LONG_DOUBLE)count; NETDATA_DOUBLE average = sum / (NETDATA_DOUBLE)count;
for(count = 0, sum = 0, value = series ; value < end ;value++) { for(count = 0, sum = 0, value = series ; value < end ;value++) {
if(calculated_number_isnumber(*value)) { if(netdata_double_isnumber(*value)) {
count++; count++;
sum += powl(*value - average, 2); sum += powndd(*value - average, 2);
} }
} }
if(unlikely(count == 0)) return NAN; if(unlikely(count == 0)) return NAN;
if(unlikely(count == 1)) return average; if(unlikely(count == 1)) return average;
LONG_DOUBLE variance = sum / (LONG_DOUBLE)(count); // remove -1 from count to have a population stddev NETDATA_DOUBLE variance = sum / (NETDATA_DOUBLE)(count); // remove -1 from count to have a population stddev
LONG_DOUBLE stddev = sqrtl(variance); NETDATA_DOUBLE stddev = sqrtndd(variance);
return stddev; return stddev;
} }
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) { NETDATA_DOUBLE single_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha) {
if(unlikely(entries == 0)) if(unlikely(entries == 0))
return NAN; return NAN;
if(unlikely(isnan(alpha))) if(unlikely(isnan(alpha)))
alpha = default_single_exponential_smoothing_alpha; alpha = default_single_exponential_smoothing_alpha;
const LONG_DOUBLE *value = series, *end = &series[entries]; const NETDATA_DOUBLE *value = series, *end = &series[entries];
LONG_DOUBLE level = (1.0 - alpha) * (*value); NETDATA_DOUBLE level = (1.0 - alpha) * (*value);
for(value++ ; value < end; value++) { for(value++ ; value < end; value++) {
if(likely(calculated_number_isnumber(*value))) if(likely(netdata_double_isnumber(*value)))
level = alpha * (*value) + (1.0 - alpha) * level; level = alpha * (*value) + (1.0 - alpha) * level;
} }
return level; return level;
} }
LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) { NETDATA_DOUBLE single_exponential_smoothing_reverse(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha) {
if(unlikely(entries == 0)) if(unlikely(entries == 0))
return NAN; return NAN;
if(unlikely(isnan(alpha))) if(unlikely(isnan(alpha)))
alpha = default_single_exponential_smoothing_alpha; alpha = default_single_exponential_smoothing_alpha;
const LONG_DOUBLE *value = &series[entries -1]; const NETDATA_DOUBLE *value = &series[entries -1];
LONG_DOUBLE level = (1.0 - alpha) * (*value); NETDATA_DOUBLE level = (1.0 - alpha) * (*value);
for(value++ ; value >= series; value--) { for(value++ ; value >= series; value--) {
if(likely(calculated_number_isnumber(*value))) if(likely(netdata_double_isnumber(*value)))
level = alpha * (*value) + (1.0 - alpha) * level; level = alpha * (*value) + (1.0 - alpha) * level;
} }
@ -260,11 +260,14 @@ LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size
// -------------------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------------------
// http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/ // http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/
LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast) { NETDATA_DOUBLE double_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries,
NETDATA_DOUBLE alpha,
NETDATA_DOUBLE beta,
NETDATA_DOUBLE *forecast) {
if(unlikely(entries == 0)) if(unlikely(entries == 0))
return NAN; return NAN;
LONG_DOUBLE level, trend; NETDATA_DOUBLE level, trend;
if(unlikely(isnan(alpha))) if(unlikely(isnan(alpha)))
alpha = 0.3; alpha = 0.3;
@ -279,11 +282,10 @@ LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entri
else else
trend = 0; trend = 0;
const LONG_DOUBLE *value = series; const NETDATA_DOUBLE *value = series;
for(value++ ; value >= series; value--) { for(value++ ; value >= series; value--) {
if(likely(calculated_number_isnumber(*value))) { if(likely(netdata_double_isnumber(*value))) {
NETDATA_DOUBLE last_level = level;
LONG_DOUBLE last_level = level;
level = alpha * *value + (1.0 - alpha) * (level + trend); level = alpha * *value + (1.0 - alpha) * (level + trend);
trend = beta * (level - last_level) + (1.0 - beta) * trend; trend = beta * (level - last_level) + (1.0 - beta) * trend;
@ -320,24 +322,26 @@ LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entri
* s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p] * s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p]
*/ */
static int __HoltWinters( static int __HoltWinters(
const LONG_DOUBLE *series, const NETDATA_DOUBLE *series,
int entries, // start_time + h int entries, // start_time + h
LONG_DOUBLE alpha, // alpha parameter of Holt-Winters Filter. NETDATA_DOUBLE alpha, // alpha parameter of Holt-Winters Filter.
LONG_DOUBLE beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing. NETDATA_DOUBLE
LONG_DOUBLE gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted. beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing.
NETDATA_DOUBLE
gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted.
const int *seasonal, const int *seasonal,
const int *period, const int *period,
const LONG_DOUBLE *a, // Start value for level (a[0]). const NETDATA_DOUBLE *a, // Start value for level (a[0]).
const LONG_DOUBLE *b, // Start value for trend (b[0]). const NETDATA_DOUBLE *b, // Start value for trend (b[0]).
LONG_DOUBLE *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0]) NETDATA_DOUBLE *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0])
/* return values */ /* return values */
LONG_DOUBLE *SSE, // The final sum of squared errors achieved in optimizing NETDATA_DOUBLE *SSE, // The final sum of squared errors achieved in optimizing
LONG_DOUBLE *level, // Estimated values for the level component (size entries - t + 2) NETDATA_DOUBLE *level, // Estimated values for the level component (size entries - t + 2)
LONG_DOUBLE *trend, // Estimated values for the trend component (size entries - t + 2) NETDATA_DOUBLE *trend, // Estimated values for the trend component (size entries - t + 2)
LONG_DOUBLE *season // Estimated values for the seasonal component (size entries - t + 2) NETDATA_DOUBLE *season // Estimated values for the seasonal component (size entries - t + 2)
) )
{ {
if(unlikely(entries < 4)) if(unlikely(entries < 4))
@ -345,13 +349,13 @@ static int __HoltWinters(
int start_time = 2; int start_time = 2;
LONG_DOUBLE res = 0, xhat = 0, stmp = 0; NETDATA_DOUBLE res = 0, xhat = 0, stmp = 0;
int i, i0, s0; int i, i0, s0;
/* copy start values to the beginning of the vectors */ /* copy start values to the beginning of the vectors */
level[0] = *a; level[0] = *a;
if(beta > 0) trend[0] = *b; if(beta > 0) trend[0] = *b;
if(gamma > 0) memcpy(season, s, *period * sizeof(LONG_DOUBLE)); if(gamma > 0) memcpy(season, s, *period * sizeof(NETDATA_DOUBLE));
for(i = start_time - 1; i < entries; i++) { for(i = start_time - 1; i < entries; i++) {
/* indices for period i */ /* indices for period i */
@ -397,7 +401,11 @@ static int __HoltWinters(
return 1; return 1;
} }
LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast) { NETDATA_DOUBLE holtwinters(const NETDATA_DOUBLE *series, size_t entries,
NETDATA_DOUBLE alpha,
NETDATA_DOUBLE beta,
NETDATA_DOUBLE gamma,
NETDATA_DOUBLE *forecast) {
if(unlikely(isnan(alpha))) if(unlikely(isnan(alpha)))
alpha = 0.3; alpha = 0.3;
@ -409,15 +417,15 @@ LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE a
int seasonal = 0; int seasonal = 0;
int period = 0; int period = 0;
LONG_DOUBLE a0 = series[0]; NETDATA_DOUBLE a0 = series[0];
LONG_DOUBLE b0 = 0; NETDATA_DOUBLE b0 = 0;
LONG_DOUBLE s[] = {}; NETDATA_DOUBLE s[] = {};
LONG_DOUBLE errors = 0.0; NETDATA_DOUBLE errors = 0.0;
size_t nb_computations = entries; size_t nb_computations = entries;
LONG_DOUBLE *estimated_level = callocz(nb_computations, sizeof(LONG_DOUBLE)); NETDATA_DOUBLE *estimated_level = callocz(nb_computations, sizeof(NETDATA_DOUBLE));
LONG_DOUBLE *estimated_trend = callocz(nb_computations, sizeof(LONG_DOUBLE)); NETDATA_DOUBLE *estimated_trend = callocz(nb_computations, sizeof(NETDATA_DOUBLE));
LONG_DOUBLE *estimated_season = callocz(nb_computations, sizeof(LONG_DOUBLE)); NETDATA_DOUBLE *estimated_season = callocz(nb_computations, sizeof(NETDATA_DOUBLE));
int ret = __HoltWinters( int ret = __HoltWinters(
series, series,
@ -436,7 +444,7 @@ LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE a
estimated_season estimated_season
); );
LONG_DOUBLE value = estimated_level[nb_computations - 1]; NETDATA_DOUBLE value = estimated_level[nb_computations - 1];
if(forecast) if(forecast)
*forecast = 0.0; *forecast = 0.0;

40
libnetdata/statistical/statistical.h
View File

@ -5,22 +5,30 @@
#include "../libnetdata.h" #include "../libnetdata.h"
extern void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg); extern void log_series_to_stderr(NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE result, const char *msg);
extern LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries); extern NETDATA_DOUBLE average(const NETDATA_DOUBLE *series, size_t entries);
extern LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period); extern NETDATA_DOUBLE moving_average(const NETDATA_DOUBLE *series, size_t entries, size_t period);
extern LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries); extern NETDATA_DOUBLE median(const NETDATA_DOUBLE *series, size_t entries);
extern LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period); extern NETDATA_DOUBLE moving_median(const NETDATA_DOUBLE *series, size_t entries, size_t period);
extern LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries); extern NETDATA_DOUBLE running_median_estimate(const NETDATA_DOUBLE *series, size_t entries);
extern LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries); extern NETDATA_DOUBLE standard_deviation(const NETDATA_DOUBLE *series, size_t entries);
extern LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha); extern NETDATA_DOUBLE single_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha);
extern LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha); extern NETDATA_DOUBLE
extern LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast); single_exponential_smoothing_reverse(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha);
extern LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast); extern NETDATA_DOUBLE double_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries,
extern LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count); NETDATA_DOUBLE alpha,
extern LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries); NETDATA_DOUBLE beta,
extern LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries); NETDATA_DOUBLE *forecast);
extern LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries); extern NETDATA_DOUBLE holtwinters(const NETDATA_DOUBLE *series, size_t entries,
extern void sort_series(LONG_DOUBLE *series, size_t entries); NETDATA_DOUBLE alpha,
NETDATA_DOUBLE beta,
NETDATA_DOUBLE gamma,
NETDATA_DOUBLE *forecast);
extern NETDATA_DOUBLE sum_and_count(const NETDATA_DOUBLE *series, size_t entries, size_t *count);
extern NETDATA_DOUBLE sum(const NETDATA_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE median_on_sorted_series(const NETDATA_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE *copy_series(const NETDATA_DOUBLE *series, size_t entries);
extern void sort_series(NETDATA_DOUBLE *series, size_t entries);
#endif //NETDATA_STATISTICAL_H #endif //NETDATA_STATISTICAL_H

34
libnetdata/storage_number/storage_number.c
View File

@ -2,7 +2,7 @@
#include "../libnetdata.h" #include "../libnetdata.h"
storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) { storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) {
// bit 32 = sign 0:positive, 1:negative // bit 32 = sign 0:positive, 1:negative
// bit 31 = 0:divide, 1:multiply // bit 31 = 0:divide, 1:multiply
// bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total) // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
@ -19,7 +19,7 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
goto RET_SN; goto RET_SN;
int m = 0; int m = 0;
calculated_number n = value, factor = 10; NETDATA_DOUBLE n = value, factor = 10;
// if the value is negative // if the value is negative
// add the sign bit and make it positive // add the sign bit and make it positive
@ -36,7 +36,7 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
// make its integer part fit in 0x00ffffff // make its integer part fit in 0x00ffffff
// by dividing it by 10 up to 7 times // by dividing it by 10 up to 7 times
// and increasing the multiplier // and increasing the multiplier
while(m < 7 && n > (calculated_number)0x00ffffff) { while(m < 7 && n > (NETDATA_DOUBLE)0x00ffffff) {
n /= factor; n /= factor;
m++; m++;
} }
@ -46,9 +46,9 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
// so we add a multiplier to unpack it // so we add a multiplier to unpack it
r += (1 << 30) + (m << 27); // the multiplier m r += (1 << 30) + (m << 27); // the multiplier m
if(n > (calculated_number)0x00ffffff) { if(n > (NETDATA_DOUBLE)0x00ffffff) {
#ifdef NETDATA_INTERNAL_CHECKS #ifdef NETDATA_INTERNAL_CHECKS
error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value); error("Number " NETDATA_DOUBLE_FORMAT " is too big.", value);
#endif #endif
r += 0x00ffffff; r += 0x00ffffff;
goto RET_SN; goto RET_SN;
@ -60,12 +60,12 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
// while the value is below 0x0019999e we can // while the value is below 0x0019999e we can
// multiply it by 10, up to 7 times, increasing // multiply it by 10, up to 7 times, increasing
// the multiplier // the multiplier
while(m < 7 && n < (calculated_number)0x0019999e) { while(m < 7 && n < (NETDATA_DOUBLE)0x0019999e) {
n *= 10; n *= 10;
m++; m++;
} }
if (unlikely(n > (calculated_number) (0x00ffffff))) { if (unlikely(n > (NETDATA_DOUBLE) (0x00ffffff))) {
n /= 10; n /= 10;
m--; m--;
} }
@ -90,7 +90,7 @@ RET_SN:
} }
// Lookup table to make storage number unpacking efficient. // Lookup table to make storage number unpacking efficient.
calculated_number unpack_storage_number_lut10x[4 * 8]; NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8];
__attribute__((constructor)) void initialize_lut(void) { __attribute__((constructor)) void initialize_lut(void) {
// The lookup table is partitioned in 4 subtables based on the // The lookup table is partitioned in 4 subtables based on the
@ -107,7 +107,7 @@ __attribute__((constructor)) void initialize_lut(void) {
} }
/* /*
int print_calculated_number(char *str, calculated_number value) int print_netdata_double(char *str, NETDATA_DOUBLE value)
{ {
char *wstr = str; char *wstr = str;
@ -117,9 +117,9 @@ int print_calculated_number(char *str, calculated_number value)
#ifdef STORAGE_WITH_MATH #ifdef STORAGE_WITH_MATH
// without llrintl() there are rounding problems // without llrintl() there are rounding problems
// for example 0.9 becomes 0.89 // for example 0.9 becomes 0.89
unsigned long long uvalue = (unsigned long long int) llrintl(value * (calculated_number)100000); unsigned long long uvalue = (unsigned long long int) llrintl(value * (NETDATA_DOUBLE)100000);
#else #else
unsigned long long uvalue = value * (calculated_number)100000; unsigned long long uvalue = value * (NETDATA_DOUBLE)100000;
#endif #endif
wstr = print_number_llu_r_smart(str, uvalue); wstr = print_number_llu_r_smart(str, uvalue);
@ -163,8 +163,8 @@ int print_calculated_number(char *str, calculated_number value)
} }
*/ */
int print_calculated_number(char *str, calculated_number value) { int print_netdata_double(char *str, NETDATA_DOUBLE value) {
// info("printing number " CALCULATED_NUMBER_FORMAT, value); // info("printing number " NETDATA_DOUBLE_FORMAT, value);
char integral_str[50], fractional_str[50]; char integral_str[50], fractional_str[50];
char *wstr = str; char *wstr = str;
@ -174,22 +174,22 @@ int print_calculated_number(char *str, calculated_number value) {
value = -value; value = -value;
} }
calculated_number integral, fractional; NETDATA_DOUBLE integral, fractional;
#ifdef STORAGE_WITH_MATH #ifdef STORAGE_WITH_MATH
fractional = calculated_number_modf(value, &integral) * 10000000.0; fractional = modfndd(value, &integral) * 10000000.0;
#else #else
fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL); fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL);
#endif #endif
unsigned long long integral_int = (unsigned long long)integral; unsigned long long integral_int = (unsigned long long)integral;
unsigned long long fractional_int = (unsigned long long)calculated_number_llrint(fractional); unsigned long long fractional_int = (unsigned long long)llrintndd(fractional);
if(unlikely(fractional_int >= 10000000)) { if(unlikely(fractional_int >= 10000000)) {
integral_int += 1; integral_int += 1;
fractional_int -= 10000000; fractional_int -= 10000000;
} }
// info("integral " CALCULATED_NUMBER_FORMAT " (%llu), fractional " CALCULATED_NUMBER_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int); // info("integral " NETDATA_DOUBLE_FORMAT " (%llu), fractional " NETDATA_DOUBLE_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int);
char *istre; char *istre;
if(unlikely(integral_int == 0)) { if(unlikely(integral_int == 0)) {

163
libnetdata/storage_number/storage_number.h
View File

@ -6,68 +6,60 @@
#include <math.h> #include <math.h>
#include "../libnetdata.h" #include "../libnetdata.h"
#ifdef NETDATA_WITHOUT_LONG_DOUBLE #ifdef NETDATA_WITH_LONG_DOUBLE
#define powl pow typedef long double NETDATA_DOUBLE;
#define modfl modf #define NETDATA_DOUBLE_FORMAT "%0.7Lf"
#define llrintl llrint #define NETDATA_DOUBLE_FORMAT_ZERO "%0.0Lf"
#define roundl round #define NETDATA_DOUBLE_FORMAT_AUTO "%Lf"
#define sqrtl sqrt #define NETDATA_DOUBLE_MODIFIER "Lf"
#define copysignl copysign
#define strtold strtod
typedef double calculated_number; #define NETDATA_DOUBLE_MAX LDBL_MAX
#define CALCULATED_NUMBER_FORMAT "%0.7f"
#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0f"
#define CALCULATED_NUMBER_FORMAT_AUTO "%f"
#define LONG_DOUBLE_MODIFIER "f" #define strtondd(s, endptr) strtold(s, endptr)
typedef double LONG_DOUBLE; #define powndd(x, y) powl(x, y)
#define llrintndd(x) llrintl(x)
#define roundndd(x) roundl(x)
#define sqrtndd(x) sqrtl(x)
#define copysignndd(x, y) copysignl(x, y)
#define modfndd(x, y) modfl(x, y)
#define fabsndd(x) fabsl(x)
#define CALCULATED_NUMBER_MAX DBL_MAX #else // NETDATA_WITH_LONG_DOUBLE
#else // NETDATA_WITHOUT_LONG_DOUBLE typedef double NETDATA_DOUBLE;
#define NETDATA_DOUBLE_FORMAT "%0.7f"
#define NETDATA_DOUBLE_FORMAT_ZERO "%0.0f"
#define NETDATA_DOUBLE_FORMAT_AUTO "%f"
#define NETDATA_DOUBLE_MODIFIER "f"
typedef long double calculated_number; #define NETDATA_DOUBLE_MAX DBL_MAX
#define CALCULATED_NUMBER_FORMAT "%0.7Lf"
#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0Lf"
#define CALCULATED_NUMBER_FORMAT_AUTO "%Lf"
#define LONG_DOUBLE_MODIFIER "Lf" #define strtondd(s, endptr) strtod(s, endptr)
typedef long double LONG_DOUBLE; #define powndd(x, y) pow(x, y)
#define llrintndd(x) llrint(x)
#define roundndd(x) round(x)
#define sqrtndd(x) sqrt(x)
#define copysignndd(x, y) copysign(x, y)
#define modfndd(x, y) modf(x, y)
#define fabsndd(x) fabs(x)
#define CALCULATED_NUMBER_MAX LDBL_MAX #endif // NETDATA_WITH_LONG_DOUBLE
#endif // NETDATA_WITHOUT_LONG_DOUBLE
//typedef long long calculated_number;
//#define CALCULATED_NUMBER_FORMAT "%lld"
typedef long long collected_number; typedef long long collected_number;
#define COLLECTED_NUMBER_FORMAT "%lld" #define COLLECTED_NUMBER_FORMAT "%lld"
/* #define epsilonndd (NETDATA_DOUBLE)0.0000001
typedef long double collected_number; #define considered_equal_ndd(a, b) (fabsndd((a) - (b)) < epsilonndd)
#define COLLECTED_NUMBER_FORMAT "%0.7Lf"
*/
#define calculated_number_modf(x, y) modfl(x, y)
#define calculated_number_llrint(x) llrintl(x)
#define calculated_number_round(x) roundl(x)
#define calculated_number_fabs(x) fabsl(x)
#define calculated_number_pow(x, y) powl(x, y)
#define calculated_number_epsilon (calculated_number)0.0000001
#define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon)
#if defined(HAVE_ISFINITE) || defined(isfinite) #if defined(HAVE_ISFINITE) || defined(isfinite)
// The isfinite() macro shall determine whether its argument has a // The isfinite() macro shall determine whether its argument has a
// finite value (zero, subnormal, or normal, and not infinite or NaN). // finite value (zero, subnormal, or normal, and not infinite or NaN).
#define calculated_number_isnumber(a) (isfinite(a)) #define netdata_double_isnumber(a) (isfinite(a))
#elif defined(HAVE_FINITE) || defined(finite) #elif defined(HAVE_FINITE) || defined(finite)
#define calculated_number_isnumber(a) (finite(a)) #define netdata_double_isnumber(a) (finite(a))
#else #else
#define calculated_number_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE) #define netdata_double_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE)
#endif #endif
typedef uint32_t storage_number; typedef uint32_t storage_number;
@ -95,10 +87,10 @@ typedef enum {
#define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT) #define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT)
#define did_storage_number_reset(value) ((((storage_number) (value)) & SN_EXISTS_RESET) != 0) #define did_storage_number_reset(value) ((((storage_number) (value)) & SN_EXISTS_RESET) != 0)
storage_number pack_storage_number(calculated_number value, SN_FLAGS flags); storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags);
static inline calculated_number unpack_storage_number(storage_number value) __attribute__((const)); static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) __attribute__((const));
int print_calculated_number(char *str, calculated_number value); int print_netdata_double(char *str, NETDATA_DOUBLE value);
// sign div/mul <--- multiplier / divider ---> 10/100 RESET EXISTS VALUE // sign div/mul <--- multiplier / divider ---> 10/100 RESET EXISTS VALUE
#define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff ) #define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff )
@ -115,8 +107,8 @@ int print_calculated_number(char *str, calculated_number value);
#define MAX_INCREMENTAL_PERCENT_RATE 10 #define MAX_INCREMENTAL_PERCENT_RATE 10
static inline calculated_number unpack_storage_number(storage_number value) { static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) {
extern calculated_number unpack_storage_number_lut10x[4 * 8]; extern NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8];
if(unlikely(value == SN_EMPTY_SLOT)) if(unlikely(value == SN_EMPTY_SLOT))
return NAN; return NAN;
@ -145,11 +137,82 @@ static inline calculated_number unpack_storage_number(storage_number value) {
// bit 24 to bit 1 = the value, so remove all other bits // bit 24 to bit 1 = the value, so remove all other bits
value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24)); value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));
calculated_number n = value; NETDATA_DOUBLE n = value;
// fprintf(stderr, "UNPACK: %08X, sign = %d, exp = %d, mul = %d, factor = %d, n = " CALCULATED_NUMBER_FORMAT "\n", value, sign, exp, mul, factor, n); // fprintf(stderr, "UNPACK: %08X, sign = %d, exp = %d, mul = %d, factor = %d, n = " CALCULATED_NUMBER_FORMAT "\n", value, sign, exp, mul, factor, n);
return sign * unpack_storage_number_lut10x[(factor * 16) + (exp * 8) + mul] * n; return sign * unpack_storage_number_lut10x[(factor * 16) + (exp * 8) + mul] * n;
} }
static inline NETDATA_DOUBLE str2ndd(const char *s, char **endptr) {
int negative = 0;
const char *start = s;
unsigned long long integer_part = 0;
unsigned long decimal_part = 0;
size_t decimal_digits = 0;
switch(*s) {
case '-':
s++;
negative = 1;
break;
case '+':
s++;
break;
case 'n':
if(s[1] == 'a' && s[2] == 'n') {
if(endptr) *endptr = (char *)&s[3];
return NAN;
}
break;
case 'i':
if(s[1] == 'n' && s[2] == 'f') {
if(endptr) *endptr = (char *)&s[3];
return INFINITY;
}
break;
default:
break;
}
while (*s >= '0' && *s <= '9') {
integer_part = (integer_part * 10) + (*s - '0');
s++;
}
if(unlikely(*s == '.')) {
decimal_part = 0;
s++;
while (*s >= '0' && *s <= '9') {
decimal_part = (decimal_part * 10) + (*s - '0');
s++;
decimal_digits++;
}
}
if(unlikely(*s == 'e' || *s == 'E'))
return strtondd(start, endptr);
if(unlikely(endptr))
*endptr = (char *)s;
if(unlikely(negative)) {
if(unlikely(decimal_digits))
return -((NETDATA_DOUBLE)integer_part + (NETDATA_DOUBLE)decimal_part / powndd(10.0, decimal_digits));
else
return -((NETDATA_DOUBLE)integer_part);
}
else {
if(unlikely(decimal_digits))
return (NETDATA_DOUBLE)integer_part + (NETDATA_DOUBLE)decimal_part / powndd(10.0, decimal_digits);
else
return (NETDATA_DOUBLE)integer_part;
}
}
#endif /* NETDATA_STORAGE_NUMBER_H */ #endif /* NETDATA_STORAGE_NUMBER_H */

20
libnetdata/storage_number/tests/test_storage_number.c
View File

@ -11,34 +11,34 @@ static void test_number_printing(void **state)
char value[50]; char value[50];
print_calculated_number(value, 0); print_netdata_double(value, 0);
assert_string_equal(value, "0"); assert_string_equal(value, "0");
print_calculated_number(value, 0.0000001); print_netdata_double(value, 0.0000001);
assert_string_equal(value, "0.0000001"); assert_string_equal(value, "0.0000001");
print_calculated_number(value, 0.00000009); print_netdata_double(value, 0.00000009);
assert_string_equal(value, "0.0000001"); assert_string_equal(value, "0.0000001");
print_calculated_number(value, 0.000000001); print_netdata_double(value, 0.000000001);
assert_string_equal(value, "0"); assert_string_equal(value, "0");
print_calculated_number(value, 99.99999999999999999); print_netdata_double(value, 99.99999999999999999);
assert_string_equal(value, "100"); assert_string_equal(value, "100");
print_calculated_number(value, -99.99999999999999999); print_netdata_double(value, -99.99999999999999999);
assert_string_equal(value, "-100"); assert_string_equal(value, "-100");
print_calculated_number(value, 123.4567890123456789); print_netdata_double(value, 123.4567890123456789);
assert_string_equal(value, "123.456789"); assert_string_equal(value, "123.456789");
print_calculated_number(value, 9999.9999999); print_netdata_double(value, 9999.9999999);
assert_string_equal(value, "9999.9999999"); assert_string_equal(value, "9999.9999999");
print_calculated_number(value, -9999.9999999); print_netdata_double(value, -9999.9999999);
assert_string_equal(value, "-9999.9999999"); assert_string_equal(value, "-9999.9999999");
print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS))); print_netdata_double(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS)));
assert_string_equal(value, "16.77722"); assert_string_equal(value, "16.77722");
} }

4
libnetdata/tests/test_str2ld.c
View File

@ -24,8 +24,8 @@ static void test_str2ld(void **state)
for (int i = 0; values[i]; i++) { for (int i = 0; values[i]; i++) {
char *e_mine = "hello", *e_sys = "world"; char *e_mine = "hello", *e_sys = "world";
LONG_DOUBLE mine = str2ld(values[i], &e_mine); NETDATA_DOUBLE mine = str2ndd(values[i], &e_mine);
LONG_DOUBLE sys = strtold(values[i], &e_sys); NETDATA_DOUBLE sys = strtondd(values[i], &e_sys);
if (isnan(mine)) if (isnan(mine))
assert_true(isnan(sys)); assert_true(isnan(sys));

2
ml/Dimension.cc
View File

@ -42,7 +42,7 @@ TrainableDimension::getCalculatedNumbers() {
auto P = Q.nextMetric(); auto P = Q.nextMetric();
CalculatedNumber Value = P.second; CalculatedNumber Value = P.second;
if (calculated_number_isnumber(Value)) { if (netdata_double_isnumber(Value)) {
CNs[Idx] = Value; CNs[Idx] = Value;
LastValue = CNs[Idx]; LastValue = CNs[Idx];
CollectedValues++; CollectedValues++;

4
ml/ml.cc
View File

@ -186,10 +186,10 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
if (MaxAnomalyRates < 1 || MaxAnomalyRates >= R->d) if (MaxAnomalyRates < 1 || MaxAnomalyRates >= R->d)
return; return;
calculated_number *CNs = R->v; NETDATA_DOUBLE *CNs = R->v;
RRDR_DIMENSION_FLAGS *DimFlags = R->od; RRDR_DIMENSION_FLAGS *DimFlags = R->od;
std::vector<std::pair<calculated_number, int>> V; std::vector<std::pair<NETDATA_DOUBLE, int>> V;
V.reserve(R->d); V.reserve(R->d);
for (int Idx = 0; Idx != R->d; Idx++) for (int Idx = 0; Idx != R->d; Idx++)

2
parser/parser.h
View File

@ -28,7 +28,7 @@ typedef struct pluginsd_action {
PARSER_RC (*flush_action)(void *user, RRDSET *st); PARSER_RC (*flush_action)(void *user, RRDSET *st);
PARSER_RC (*disable_action)(void *user); PARSER_RC (*disable_action)(void *user);
PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value); PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, NETDATA_DOUBLE value);
PARSER_RC (*label_action)(void *user, char *key, char *value, RRDLABEL_SRC source); PARSER_RC (*label_action)(void *user, char *key, char *value, RRDLABEL_SRC source);
PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, DICTIONARY *new_labels); PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, DICTIONARY *new_labels);
PARSER_RC (*clabel_action)(void *user, char *key, char *value, RRDLABEL_SRC source); PARSER_RC (*clabel_action)(void *user, char *key, char *value, RRDLABEL_SRC source);

13
streaming/rrdpush.c
View File

@ -75,7 +75,8 @@ int rrdpush_init() {
default_rrdpush_destination = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "destination", ""); default_rrdpush_destination = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "destination", "");
default_rrdpush_api_key = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "api key", ""); default_rrdpush_api_key = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "api key", "");
default_rrdpush_send_charts_matching = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "send charts matching", "*"); default_rrdpush_send_charts_matching = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "send charts matching", "*");
rrdhost_free_orphan_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup orphan hosts after seconds", rrdhost_free_orphan_time); rrdhost_free_orphan_time = config_get_number(CONFIG_SECTION_DB, "cleanup orphan hosts after secs", rrdhost_free_orphan_time);
#ifdef ENABLE_COMPRESSION #ifdef ENABLE_COMPRESSION
default_compression_enabled = (unsigned int)appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM, default_compression_enabled = (unsigned int)appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM,
"enable compression", default_compression_enabled); "enable compression", default_compression_enabled);
@ -97,10 +98,10 @@ int rrdpush_init() {
} }
} }
char *invalid_certificate = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "ssl skip certificate verification", "no"); bool invalid_certificate = appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM, "ssl skip certificate verification", CONFIG_BOOLEAN_NO);
if ( !strcmp(invalid_certificate,"yes")){ if(invalid_certificate == CONFIG_BOOLEAN_YES){
if (netdata_validate_server == NETDATA_SSL_VALID_CERTIFICATE){ if(netdata_validate_server == NETDATA_SSL_VALID_CERTIFICATE){
info("Netdata is configured to accept invalid SSL certificate."); info("Netdata is configured to accept invalid SSL certificate.");
netdata_validate_server = NETDATA_SSL_INVALID_CERTIFICATE; netdata_validate_server = NETDATA_SSL_INVALID_CERTIFICATE;
} }
@ -272,11 +273,11 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
RRDSETVAR *rs; RRDSETVAR *rs;
for(rs = st->variables; rs ;rs = rs->next) { for(rs = st->variables; rs ;rs = rs->next) {
if(unlikely(rs->type == RRDVAR_TYPE_CALCULATED && rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR)) { if(unlikely(rs->type == RRDVAR_TYPE_CALCULATED && rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR)) {
calculated_number *value = (calculated_number *) rs->value; NETDATA_DOUBLE *value = (NETDATA_DOUBLE *) rs->value;
buffer_sprintf( buffer_sprintf(
host->sender->build host->sender->build
, "VARIABLE CHART %s = " CALCULATED_NUMBER_FORMAT "\n" , "VARIABLE CHART %s = " NETDATA_DOUBLE_FORMAT "\n"
, rs->variable , rs->variable
, *value , *value
); );

6
streaming/sender.c
View File

@ -89,16 +89,16 @@ static inline void rrdpush_sender_thread_close_socket(RRDHOST *host) {
} }
static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *host, RRDVAR *rv) { static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *host, RRDVAR *rv) {
calculated_number *value = (calculated_number *)rv->value; NETDATA_DOUBLE *value = (NETDATA_DOUBLE *)rv->value;
buffer_sprintf( buffer_sprintf(
host->sender->build host->sender->build
, "VARIABLE HOST %s = " CALCULATED_NUMBER_FORMAT "\n" , "VARIABLE HOST %s = " NETDATA_DOUBLE_FORMAT "\n"
, rv->name , rv->name
, *value , *value
); );
debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " CALCULATED_NUMBER_FORMAT, rv->name, *value); debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rv->name, *value);
} }
void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) { void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) {

10
tests/profile/test-eval.c
View File

@ -58,9 +58,9 @@ void print_node(EVAL_NODE *op, int level) {
while(i--) print_value(&op->ops[i], level + 1); while(i--) print_value(&op->ops[i], level + 1);
} }
calculated_number evaluate(EVAL_NODE *op, int depth); NETDATA_DOUBLE evaluate(EVAL_NODE *op, int depth);
calculated_number evaluate_value(EVAL_VALUE *v, int depth) { NETDATA_DOUBLE evaluate_value(EVAL_VALUE *v, int depth) {
switch(v->type) { switch(v->type) {
case EVAL_VALUE_NUMBER: case EVAL_VALUE_NUMBER:
return v->number; return v->number;
@ -80,8 +80,8 @@ void print_depth(int depth) {
while(depth--) printf(" "); while(depth--) printf(" ");
} }
calculated_number evaluate(EVAL_NODE *op, int depth) { NETDATA_DOUBLE evaluate(EVAL_NODE *op, int depth) {
calculated_number n1, n2, r; NETDATA_DOUBLE n1, n2, r;
switch(op->operator) { switch(op->operator) {
case EVAL_OPERATOR_SIGN_PLUS: case EVAL_OPERATOR_SIGN_PLUS:
@ -249,7 +249,7 @@ void print_expression(EVAL_NODE *op, const char *failed_at, int error) {
evaluate(op, 0); evaluate(op, 0);
int error; int error;
calculated_number ret = expression_evaluate(op, &error); NETDATA_DOUBLE ret = expression_evaluate(op, &error);
printf("\ninternal evaluator:\nSTATUS: %d, RESULT = %Lf\n", error, ret); printf("\ninternal evaluator:\nSTATUS: %d, RESULT = %Lf\n", error, ret);
expression_free(op); expression_free(op);

39
web/api/badges/web_buffer_svg.c
View File

@ -249,7 +249,8 @@ cleanup:
return len - i; return len - i;
} }
static inline char *format_value_with_precision_and_unit(char *value_string, size_t value_string_len, calculated_number value, const char *units, int precision) { static inline char *format_value_with_precision_and_unit(char *value_string, size_t value_string_len,
NETDATA_DOUBLE value, const char *units, int precision) {
if(unlikely(isnan(value) || isinf(value))) if(unlikely(isnan(value) || isinf(value)))
value = 0.0; value = 0.0;
@ -260,23 +261,23 @@ static inline char *format_value_with_precision_and_unit(char *value_string, siz
if(precision < 0) { if(precision < 0) {
int len, lstop = 0, trim_zeros = 1; int len, lstop = 0, trim_zeros = 1;
calculated_number abs = value; NETDATA_DOUBLE abs = value;
if(isless(value, 0)) { if(isless(value, 0)) {
lstop = 1; lstop = 1;
abs = calculated_number_fabs(value); abs = fabsndd(value);
} }
if(isgreaterequal(abs, 1000)) { if(isgreaterequal(abs, 1000)) {
len = snprintfz(value_string, value_string_len, "%0.0" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value); len = snprintfz(value_string, value_string_len, "%0.0" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
trim_zeros = 0; trim_zeros = 0;
} }
else if(isgreaterequal(abs, 10)) len = snprintfz(value_string, value_string_len, "%0.1" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value); else if(isgreaterequal(abs, 10)) len = snprintfz(value_string, value_string_len, "%0.1" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 1)) len = snprintfz(value_string, value_string_len, "%0.2" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value); else if(isgreaterequal(abs, 1)) len = snprintfz(value_string, value_string_len, "%0.2" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 0.1)) len = snprintfz(value_string, value_string_len, "%0.2" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value); else if(isgreaterequal(abs, 0.1)) len = snprintfz(value_string, value_string_len, "%0.2" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 0.01)) len = snprintfz(value_string, value_string_len, "%0.4" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value); else if(isgreaterequal(abs, 0.01)) len = snprintfz(value_string, value_string_len, "%0.4" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 0.001)) len = snprintfz(value_string, value_string_len, "%0.5" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value); else if(isgreaterequal(abs, 0.001)) len = snprintfz(value_string, value_string_len, "%0.5" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 0.0001)) len = snprintfz(value_string, value_string_len, "%0.6" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value); else if(isgreaterequal(abs, 0.0001)) len = snprintfz(value_string, value_string_len, "%0.6" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else len = snprintfz(value_string, value_string_len, "%0.7" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value); else len = snprintfz(value_string, value_string_len, "%0.7" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
if(unlikely(trim_zeros)) { if(unlikely(trim_zeros)) {
int l; int l;
@ -303,7 +304,7 @@ static inline char *format_value_with_precision_and_unit(char *value_string, siz
} }
else { else {
if(precision > 50) precision = 50; if(precision > 50) precision = 50;
snprintfz(value_string, value_string_len, "%0.*" LONG_DOUBLE_MODIFIER "%s%s", precision, (LONG_DOUBLE) value, separator, units); snprintfz(value_string, value_string_len, "%0.*" NETDATA_DOUBLE_MODIFIER "%s%s", precision, (NETDATA_DOUBLE) value, separator, units);
} }
return value_string; return value_string;
@ -359,7 +360,8 @@ static struct units_formatter {
{ NULL, 0, UNITS_FORMAT_NONE } { NULL, 0, UNITS_FORMAT_NONE }
}; };
inline char *format_value_and_unit(char *value_string, size_t value_string_len, calculated_number value, const char *units, int precision) { inline char *format_value_and_unit(char *value_string, size_t value_string_len,
NETDATA_DOUBLE value, const char *units, int precision) {
static int max = -1; static int max = -1;
int i; int i;
@ -555,7 +557,7 @@ typedef enum color_comparison {
COLOR_COMPARE_GREATEREQUAL, COLOR_COMPARE_GREATEREQUAL,
} BADGE_COLOR_COMPARISON; } BADGE_COLOR_COMPARISON;
static inline void calc_colorz(const char *color, char *final, size_t len, calculated_number value) { static inline void calc_colorz(const char *color, char *final, size_t len, NETDATA_DOUBLE value) {
if(isnan(value) || isinf(value)) if(isnan(value) || isinf(value))
value = NAN; value = NAN;
@ -642,7 +644,7 @@ static inline void calc_colorz(const char *color, char *final, size_t len, calcu
*dc = '\0'; *dc = '\0';
if(dv) { if(dv) {
*dv = '\0'; *dv = '\0';
calculated_number v; NETDATA_DOUBLE v;
if(!*value_buffer || !strcmp(value_buffer, "null")) { if(!*value_buffer || !strcmp(value_buffer, "null")) {
v = NAN; v = NAN;
@ -732,7 +734,8 @@ static const char *parse_color_argument(const char *arg, const char *def)
return color_map(arg, def); return color_map(arg, def);
} }
void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val) { void buffer_svg(BUFFER *wb, const char *label,
NETDATA_DOUBLE value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val) {
char value_color_buffer[COLOR_STRING_SIZE + 1] char value_color_buffer[COLOR_STRING_SIZE + 1]
, value_string[VALUE_STRING_SIZE + 1] , value_string[VALUE_STRING_SIZE + 1]
, label_escaped[LABEL_STRING_SIZE + 1] , label_escaped[LABEL_STRING_SIZE + 1]
@ -750,7 +753,7 @@ void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const ch
value_color = (isnan(value) || isinf(value))?"999":"4c1"; value_color = (isnan(value) || isinf(value))?"999":"4c1";
calc_colorz(value_color, value_color_buffer, COLOR_STRING_SIZE, value); calc_colorz(value_color, value_color_buffer, COLOR_STRING_SIZE, value);
format_value_and_unit(value_string, VALUE_STRING_SIZE, (options & RRDR_OPTION_DISPLAY_ABS)?calculated_number_fabs(value):value, units, precision); format_value_and_unit(value_string, VALUE_STRING_SIZE, (options & RRDR_OPTION_DISPLAY_ABS)? fabsndd(value):value, units, precision);
if(fixed_width_lbl <= 0 || fixed_width_val <= 0) { if(fixed_width_lbl <= 0 || fixed_width_val <= 0) {
label_width = verdana11_width(label, font_size) + (BADGE_HORIZONTAL_PADDING * 2); label_width = verdana11_width(label, font_size) + (BADGE_HORIZONTAL_PADDING * 2);
@ -1098,7 +1101,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
else { else {
time_t latest_timestamp = 0; time_t latest_timestamp = 0;
int value_is_null = 1; int value_is_null = 1;
calculated_number n = NAN; NETDATA_DOUBLE n = NAN;
ret = HTTP_RESP_INTERNAL_SERVER_ERROR; ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
// if the collected value is too old, don't calculate its value // if the collected value is too old, don't calculate its value

6
web/api/badges/web_buffer_svg.h
View File

@ -6,8 +6,10 @@
#include "libnetdata/libnetdata.h" #include "libnetdata/libnetdata.h"
#include "web/server/web_client.h" #include "web/server/web_client.h"
extern void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val); extern void buffer_svg(BUFFER *wb, const char *label,
extern char *format_value_and_unit(char *value_string, size_t value_string_len, calculated_number value, const char *units, int precision); NETDATA_DOUBLE value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val);
extern char *format_value_and_unit(char *value_string, size_t value_string_len,
NETDATA_DOUBLE value, const char *units, int precision);
extern int web_client_api_request_v1_badge(struct rrdhost *host, struct web_client *w, char *url); extern int web_client_api_request_v1_badge(struct rrdhost *host, struct web_client *w, char *url);

20
web/api/exporters/shell/allmetrics_shell.c
View File

@ -33,7 +33,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
if (filter && !simple_pattern_matches(filter, st->name)) if (filter && !simple_pattern_matches(filter, st->name))
continue; continue;
calculated_number total = 0.0; NETDATA_DOUBLE total = 0.0;
char chart[SHELL_ELEMENT_MAX + 1]; char chart[SHELL_ELEMENT_MAX + 1];
shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX); shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX);
@ -48,21 +48,21 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
char dimension[SHELL_ELEMENT_MAX + 1]; char dimension[SHELL_ELEMENT_MAX + 1];
shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX); shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX);
calculated_number n = rd->last_stored_value; NETDATA_DOUBLE n = rd->last_stored_value;
if(isnan(n) || isinf(n)) if(isnan(n) || isinf(n))
buffer_sprintf(wb, "NETDATA_%s_%s=\"\" # %s\n", chart, dimension, st->units); buffer_sprintf(wb, "NETDATA_%s_%s=\"\" # %s\n", chart, dimension, st->units);
else { else {
if(rd->multiplier < 0 || rd->divisor < 0) n = -n; if(rd->multiplier < 0 || rd->divisor < 0) n = -n;
n = calculated_number_round(n); n = roundndd(n);
if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n; if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n;
buffer_sprintf(wb, "NETDATA_%s_%s=\"" CALCULATED_NUMBER_FORMAT_ZERO "\" # %s\n", chart, dimension, n, st->units); buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, dimension, n, st->units);
} }
} }
} }
total = calculated_number_round(total); total = roundndd(total);
buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" CALCULATED_NUMBER_FORMAT_ZERO "\" # %s\n", chart, total, st->units); buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, total, st->units);
rrdset_unlock(st); rrdset_unlock(st);
} }
} }
@ -79,13 +79,13 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
char alarm[SHELL_ELEMENT_MAX + 1]; char alarm[SHELL_ELEMENT_MAX + 1];
shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX); shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX);
calculated_number n = rc->value; NETDATA_DOUBLE n = rc->value;
if(isnan(n) || isinf(n)) if(isnan(n) || isinf(n))
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\" # %s\n", chart, alarm, rc->units); buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\" # %s\n", chart, alarm, rc->units);
else { else {
n = calculated_number_round(n); n = roundndd(n);
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" CALCULATED_NUMBER_FORMAT_ZERO "\" # %s\n", chart, alarm, n, rc->units); buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, alarm, n, rc->units);
} }
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status)); buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status));
@ -154,7 +154,7 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
if(isnan(rd->last_stored_value)) if(isnan(rd->last_stored_value))
buffer_strcat(wb, "null"); buffer_strcat(wb, "null");
else else
buffer_sprintf(wb, CALCULATED_NUMBER_FORMAT, rd->last_stored_value); buffer_sprintf(wb, NETDATA_DOUBLE_FORMAT, rd->last_stored_value);
buffer_strcat(wb, "\n\t\t\t}"); buffer_strcat(wb, "\n\t\t\t}");

10
web/api/formatters/csv/csv.c
View File

@ -63,9 +63,9 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
} }
// for each line in the array // for each line in the array
calculated_number total = 1; NETDATA_DOUBLE total = 1;
for(i = start; i != end ;i += step) { for(i = start; i != end ;i += step) {
calculated_number *cn = &r->v[ i * r->d ]; NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
buffer_strcat(wb, betweenlines); buffer_strcat(wb, betweenlines);
@ -75,7 +75,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
// print the timestamp of the line // print the timestamp of the line
buffer_rrd_value(wb, (calculated_number)now); buffer_rrd_value(wb, (NETDATA_DOUBLE)now);
// in ms // in ms
if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
} }
@ -90,7 +90,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0; total = 0;
for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
calculated_number n = cn[c]; NETDATA_DOUBLE n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n; n = -n;
@ -109,7 +109,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
buffer_strcat(wb, separator); buffer_strcat(wb, separator);
calculated_number n = cn[c]; NETDATA_DOUBLE n = cn[c];
if(co[c] & RRDR_VALUE_EMPTY) { if(co[c] & RRDR_VALUE_EMPTY) {
if(options & RRDR_OPTION_NULL2ZERO) if(options & RRDR_OPTION_NULL2ZERO)

14
web/api/formatters/json/json.c
View File

@ -158,9 +158,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
); );
// for each line in the array // for each line in the array
calculated_number total = 1; NETDATA_DOUBLE total = 1;
for(i = start; i != end ;i += step) { for(i = start; i != end ;i += step) {
calculated_number *cn = &r->v[ i * r->d ]; NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
uint8_t *ar = &r->ar[ i * r->d ]; uint8_t *ar = &r->ar[ i * r->d ];
@ -210,7 +210,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); buffer_fast_strcat(wb, object_rows_time, object_rows_time_len);
buffer_rrd_value(wb, (calculated_number)r->t[i]); buffer_rrd_value(wb, (NETDATA_DOUBLE)r->t[i]);
// in ms // in ms
if(unlikely(options & RRDR_OPTION_MILLISECONDS)) if(unlikely(options & RRDR_OPTION_MILLISECONDS))
@ -223,9 +223,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0; total = 0;
for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
calculated_number n; NETDATA_DOUBLE n;
if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
n = (calculated_number)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200 n = (NETDATA_DOUBLE)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200
else else
n = cn[c]; n = cn[c];
@ -244,9 +244,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
calculated_number n; NETDATA_DOUBLE n;
if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
n = (calculated_number)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200 n = (NETDATA_DOUBLE)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200
else else
n = cn[c]; n = cn[c];

12
web/api/formatters/json_wrapper.c
View File

@ -257,7 +257,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if(i) buffer_strcat(wb, ", "); if(i) buffer_strcat(wb, ", ");
i++; i++;
calculated_number value = rd->last_stored_value; NETDATA_DOUBLE value = rd->last_stored_value;
if (NAN == value) if (NAN == value)
buffer_strcat(wb, "null"); buffer_strcat(wb, "null");
else else
@ -282,13 +282,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
i = 0; i = 0;
if(rows) { if(rows) {
calculated_number total = 1; NETDATA_DOUBLE total = 1;
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0; total = 0;
for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
calculated_number n = cn[c]; NETDATA_DOUBLE n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n; n = -n;
@ -306,9 +306,9 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if(i) buffer_strcat(wb, ", "); if(i) buffer_strcat(wb, ", ");
i++; i++;
calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ];
calculated_number n = cn[c]; NETDATA_DOUBLE n = cn[c];
if(co[c] & RRDR_VALUE_EMPTY) { if(co[c] & RRDR_VALUE_EMPTY) {
if(options & RRDR_OPTION_NULL2ZERO) if(options & RRDR_OPTION_NULL2ZERO)

4
web/api/formatters/rrd2json.c
View File

@ -86,7 +86,7 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis
(*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st)); (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st));
rrddim_foreach_read(rd1, st) { rrddim_foreach_read(rd1, st) {
RRDDIM *rd = onewayalloc_memdupz(owa, rd1, rd1->memsize); RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM));
rd->id = onewayalloc_strdupz(owa, rd1->id); rd->id = onewayalloc_strdupz(owa, rd1->id);
rd->name = onewayalloc_strdupz(owa, rd1->name); rd->name = onewayalloc_strdupz(owa, rd1->name);
rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state)); rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state));
@ -152,7 +152,7 @@ void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) {
int rrdset2value_api_v1( int rrdset2value_api_v1(
RRDSET *st RRDSET *st
, BUFFER *wb , BUFFER *wb
, calculated_number *n , NETDATA_DOUBLE *n
, const char *dimensions , const char *dimensions
, long points , long points
, long long after , long long after

6
web/api/formatters/rrd2json.h
View File

@ -67,8 +67,8 @@ extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
extern int rrdset2anything_api_v1( extern int rrdset2anything_api_v1(
ONEWAYALLOC *owa ONEWAYALLOC *owa
, RRDSET *st , RRDSET *st
, , QUERY_PARAMS *query_params
QUERY_PARAMS *query_params, BUFFER *dimensions , BUFFER *dimensions
, uint32_t format , uint32_t format
, long points , long points
, long long after , long long after
@ -83,7 +83,7 @@ extern int rrdset2anything_api_v1(
extern int rrdset2value_api_v1( extern int rrdset2value_api_v1(
RRDSET *st RRDSET *st
, BUFFER *wb , BUFFER *wb
, calculated_number *n , NETDATA_DOUBLE *n
, const char *dimensions , const char *dimensions
, long points , long points
, long long after , long long after

4
web/api/formatters/rrdset2json.c
View File

@ -85,14 +85,14 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
"\t\t\t\"dimensions\": {\n", "\t\t\t\"dimensions\": {\n",
st->update_every); st->update_every);
unsigned long memory = st->memsize; unsigned long memory = sizeof(RRDSET) + st->memsize;
size_t dimensions = 0; size_t dimensions = 0;
RRDDIM *rd; RRDDIM *rd;
rrddim_foreach_read(rd, st) { rrddim_foreach_read(rd, st) {
if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue;
memory += rd->memsize; memory += sizeof(RRDDIM) + rd->memsize;
if (dimensions) if (dimensions)
buffer_strcat(wb, ",\n\t\t\t\t\""); buffer_strcat(wb, ",\n\t\t\t\t\"");

2
web/api/formatters/ssv/ssv.c
View File

@ -17,7 +17,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con
// for each line in the array // for each line in the array
for(i = start; i != end ;i += step) { for(i = start; i != end ;i += step) {
int all_values_are_null = 0; int all_values_are_null = 0;
calculated_number v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd); NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd);
if(likely(i != start)) { if(likely(i != start)) {
if(r->min > v) r->min = v; if(r->min > v) r->min = v;

13
web/api/formatters/value/value.c
View File

@ -3,28 +3,29 @@
#include "value.h" #include "value.h"
inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd) { inline NETDATA_DOUBLE
rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd) {
if (r->st_needs_lock) if (r->st_needs_lock)
rrdset_check_rdlock(r->st); rrdset_check_rdlock(r->st);
long c; long c;
RRDDIM *d; RRDDIM *d;
calculated_number *cn = &r->v[ i * r->d ]; NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
uint8_t *ar = &r->ar[ i * r->d ]; uint8_t *ar = &r->ar[ i * r->d ];
calculated_number sum = 0, min = 0, max = 0, v; NETDATA_DOUBLE sum = 0, min = 0, max = 0, v;
int all_null = 1, init = 1; int all_null = 1, init = 1;
calculated_number total = 1; NETDATA_DOUBLE total = 1;
size_t total_anomaly_rate = 0; size_t total_anomaly_rate = 0;
int set_min_max = 0; int set_min_max = 0;
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0; total = 0;
for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) { for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) {
calculated_number n = cn[c]; NETDATA_DOUBLE n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n; n = -n;
@ -41,7 +42,7 @@ inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
calculated_number n = cn[c]; NETDATA_DOUBLE n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n; n = -n;

3
web/api/formatters/value/value.h
View File

@ -5,6 +5,7 @@
#include "../rrd2json.h" #include "../rrd2json.h"
extern calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd); extern NETDATA_DOUBLE
rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd);
#endif //NETDATA_API_FORMATTER_VALUE_H #endif //NETDATA_API_FORMATTER_VALUE_H

8
web/api/queries/average/average.c
View File

@ -6,7 +6,7 @@
// average // average
struct grouping_average { struct grouping_average {
calculated_number sum; NETDATA_DOUBLE sum;
size_t count; size_t count;
}; };
@ -27,16 +27,16 @@ void grouping_free_average(RRDR *r) {
r->internal.grouping_data = NULL; r->internal.grouping_data = NULL;
} }
void grouping_add_average(RRDR *r, calculated_number value) { void grouping_add_average(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
g->sum += value; g->sum += value;
g->count++; g->count++;
} }
calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
calculated_number value; NETDATA_DOUBLE value;
if(unlikely(!g->count)) { if(unlikely(!g->count)) {
value = 0.0; value = 0.0;

4
web/api/queries/average/average.h
View File

@ -9,7 +9,7 @@
extern void grouping_create_average(RRDR *r, const char *options __maybe_unused); extern void grouping_create_average(RRDR *r, const char *options __maybe_unused);
extern void grouping_reset_average(RRDR *r); extern void grouping_reset_average(RRDR *r);
extern void grouping_free_average(RRDR *r); extern void grouping_free_average(RRDR *r);
extern void grouping_add_average(RRDR *r, calculated_number value); extern void grouping_add_average(RRDR *r, NETDATA_DOUBLE value);
extern calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); extern NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_AVERAGE_H #endif //NETDATA_API_QUERY_AVERAGE_H

26
web/api/queries/countif/countif.c
View File

@ -6,33 +6,33 @@
// countif // countif
struct grouping_countif { struct grouping_countif {
size_t (*comparison)(calculated_number, calculated_number); size_t (*comparison)(NETDATA_DOUBLE, NETDATA_DOUBLE);
calculated_number target; NETDATA_DOUBLE target;
size_t count; size_t count;
size_t matched; size_t matched;
}; };
static size_t countif_equal(calculated_number v, calculated_number target) { static size_t countif_equal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v == target); return (v == target);
} }
static size_t countif_notequal(calculated_number v, calculated_number target) { static size_t countif_notequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v != target); return (v != target);
} }
static size_t countif_less(calculated_number v, calculated_number target) { static size_t countif_less(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v < target); return (v < target);
} }
static size_t countif_lessequal(calculated_number v, calculated_number target) { static size_t countif_lessequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v <= target); return (v <= target);
} }
static size_t countif_greater(calculated_number v, calculated_number target) { static size_t countif_greater(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v > target); return (v > target);
} }
static size_t countif_greaterequal(calculated_number v, calculated_number target) { static size_t countif_greaterequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v >= target); return (v >= target);
} }
@ -89,7 +89,7 @@ void grouping_create_countif(RRDR *r, const char *options __maybe_unused) {
// skip everything up to the first digit // skip everything up to the first digit
while(isspace(*options)) options++; while(isspace(*options)) options++;
g->target = str2ld(options, NULL); g->target = str2ndd(options, NULL);
} }
else { else {
g->target = 0.0; g->target = 0.0;
@ -110,23 +110,23 @@ void grouping_free_countif(RRDR *r) {
r->internal.grouping_data = NULL; r->internal.grouping_data = NULL;
} }
void grouping_add_countif(RRDR *r, calculated_number value) { void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data; struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;
g->matched += g->comparison(value, g->target); g->matched += g->comparison(value, g->target);
g->count++; g->count++;
} }
calculated_number grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data; struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;
calculated_number value; NETDATA_DOUBLE value;
if(unlikely(!g->count)) { if(unlikely(!g->count)) {
value = 0.0; value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
} }
else { else {
value = (calculated_number)g->matched * 100 / (calculated_number)g->count; value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count;
} }
g->matched = 0; g->matched = 0;

4
web/api/queries/countif/countif.h
View File

@ -9,7 +9,7 @@
extern void grouping_create_countif(RRDR *r, const char *options __maybe_unused); extern void grouping_create_countif(RRDR *r, const char *options __maybe_unused);
extern void grouping_reset_countif(RRDR *r); extern void grouping_reset_countif(RRDR *r);
extern void grouping_free_countif(RRDR *r); extern void grouping_free_countif(RRDR *r);
extern void grouping_add_countif(RRDR *r, calculated_number value); extern void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value);
extern calculated_number grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); extern NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_COUNTIF_H #endif //NETDATA_API_QUERY_COUNTIF_H

24
web/api/queries/des/des.c
View File

@ -8,13 +8,13 @@
// single exponential smoothing // single exponential smoothing
struct grouping_des { struct grouping_des {
calculated_number alpha; NETDATA_DOUBLE alpha;
calculated_number alpha_other; NETDATA_DOUBLE alpha_other;
calculated_number beta; NETDATA_DOUBLE beta;
calculated_number beta_other; NETDATA_DOUBLE beta_other;
calculated_number level; NETDATA_DOUBLE level;
calculated_number trend; NETDATA_DOUBLE trend;
size_t count; size_t count;
}; };
@ -31,10 +31,10 @@ void grouping_init_des(void) {
} }
} }
static inline calculated_number window(RRDR *r, struct grouping_des *g) { static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_des *g) {
(void)g; (void)g;
calculated_number points; NETDATA_DOUBLE points;
if(r->group == 1) { if(r->group == 1) {
// provide a running DES // provide a running DES
points = r->internal.points_wanted; points = r->internal.points_wanted;
@ -96,7 +96,7 @@ void grouping_free_des(RRDR *r) {
r->internal.grouping_data = NULL; r->internal.grouping_data = NULL;
} }
void grouping_add_des(RRDR *r, calculated_number value) { void grouping_add_des(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data; struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
if(likely(g->count > 0)) { if(likely(g->count > 0)) {
@ -109,7 +109,7 @@ void grouping_add_des(RRDR *r, calculated_number value) {
} }
// for the values, except the first // for the values, except the first
calculated_number last_level = g->level; NETDATA_DOUBLE last_level = g->level;
g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend)); g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend));
g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend); g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend);
} }
@ -123,10 +123,10 @@ void grouping_add_des(RRDR *r, calculated_number value) {
//fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend); //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend);
} }
calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data; struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
if(unlikely(!g->count || !calculated_number_isnumber(g->level))) { if(unlikely(!g->count || !netdata_double_isnumber(g->level))) {
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
return 0.0; return 0.0;
} }

4
web/api/queries/des/des.h
View File

@ -11,7 +11,7 @@ extern void grouping_init_des(void);
extern void grouping_create_des(RRDR *r, const char *options __maybe_unused); extern void grouping_create_des(RRDR *r, const char *options __maybe_unused);
extern void grouping_reset_des(RRDR *r); extern void grouping_reset_des(RRDR *r);
extern void grouping_free_des(RRDR *r); extern void grouping_free_des(RRDR *r);
extern void grouping_add_des(RRDR *r, calculated_number value); extern void grouping_add_des(RRDR *r, NETDATA_DOUBLE value);
extern calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); extern NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_DES_H #endif //NETDATA_API_QUERIES_DES_H

10
web/api/queries/incremental_sum/incremental_sum.c
View File

@ -6,8 +6,8 @@
// incremental sum // incremental sum
struct grouping_incremental_sum { struct grouping_incremental_sum {
calculated_number first; NETDATA_DOUBLE first;
calculated_number last; NETDATA_DOUBLE last;
size_t count; size_t count;
}; };
@ -29,7 +29,7 @@ void grouping_free_incremental_sum(RRDR *r) {
r->internal.grouping_data = NULL; r->internal.grouping_data = NULL;
} }
void grouping_add_incremental_sum(RRDR *r, calculated_number value) { void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
if(unlikely(!g->count)) { if(unlikely(!g->count)) {
@ -42,10 +42,10 @@ void grouping_add_incremental_sum(RRDR *r, calculated_number value) {
} }
} }
calculated_number grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
calculated_number value; NETDATA_DOUBLE value;
if(unlikely(!g->count)) { if(unlikely(!g->count)) {
value = 0.0; value = 0.0;

4
web/api/queries/incremental_sum/incremental_sum.h
View File

@ -9,7 +9,7 @@
extern void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused); extern void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused);
extern void grouping_reset_incremental_sum(RRDR *r); extern void grouping_reset_incremental_sum(RRDR *r);
extern void grouping_free_incremental_sum(RRDR *r); extern void grouping_free_incremental_sum(RRDR *r);
extern void grouping_add_incremental_sum(RRDR *r, calculated_number value); extern void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value);
extern calculated_number grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); extern NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H #endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H

10
web/api/queries/max/max.c
View File

@ -6,7 +6,7 @@
// max // max
struct grouping_max { struct grouping_max {
calculated_number max; NETDATA_DOUBLE max;
size_t count; size_t count;
}; };
@ -27,19 +27,19 @@ void grouping_free_max(RRDR *r) {
r->internal.grouping_data = NULL; r->internal.grouping_data = NULL;
} }
void grouping_add_max(RRDR *r, calculated_number value) { void grouping_add_max(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
if(!g->count || calculated_number_fabs(value) > calculated_number_fabs(g->max)) { if(!g->count || fabsndd(value) > fabsndd(g->max)) {
g->max = value; g->max = value;
g->count++; g->count++;
} }
} }
calculated_number grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
calculated_number value; NETDATA_DOUBLE value;
if(unlikely(!g->count)) { if(unlikely(!g->count)) {
value = 0.0; value = 0.0;

Some files were not shown because too many files have changed in this diff Show More