opensourcemirror
/
netdata
mirror of https://github.com/netdata/netdata.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Add constant page type

This commit is contained in:
vkalintiris 2024-03-19 14:27:03 +02:00
parent f70556d7cb
commit cad1692853
10 changed files with 203 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/daemon/global_statistics.c
View File

@ -70,6 +70,9 @@ static struct global_statistics {
uint64_t tier0_disk_compressed_bytes;
uint64_t tier0_disk_uncompressed_bytes;
uint64_t tier0_raw_pages;
uint64_t tier0_constant_pages;
uint64_t db_points_stored_per_tier[RRD_STORAGE_TIERS];
} global_statistics = {
@ -89,6 +92,9 @@ static struct global_statistics {
.tier0_hot_gorilla_buffers = 0,
.tier0_disk_compressed_bytes = 0,
.tier0_disk_uncompressed_bytes = 0,
.tier0_raw_pages = 0,
.tier0_constant_pages = 0,
};
void global_statistics_rrdset_done_chart_collection_completed(size_t *points_read_per_tier_array) {
@ -129,6 +135,18 @@ void global_statistics_tier0_disk_uncompressed_bytes(uint32_t size) {
__atomic_fetch_add(&global_statistics.tier0_disk_uncompressed_bytes, size, __ATOMIC_RELAXED);
}
void global_statistic_raw_page_new() {
__atomic_fetch_add(&global_statistics.tier0_raw_pages, 1, __ATOMIC_RELAXED);
}
void global_statistic_constant_page_new() {
__atomic_fetch_add(&global_statistics.tier0_constant_pages, 1, __ATOMIC_RELAXED);
}
void global_statistic_constant_page_rm() {
__atomic_fetch_sub(&global_statistics.tier0_constant_pages, 1, __ATOMIC_RELAXED);
}
void global_statistics_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source) {
switch(query_source) {
case QUERY_SOURCE_API_DATA:
@ -236,6 +254,9 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
gs->tier0_disk_compressed_bytes = __atomic_load_n(&global_statistics.tier0_disk_compressed_bytes, __ATOMIC_RELAXED);
gs->tier0_disk_uncompressed_bytes = __atomic_load_n(&global_statistics.tier0_disk_uncompressed_bytes, __ATOMIC_RELAXED);
gs->tier0_raw_pages = __atomic_load_n(&global_statistics.tier0_raw_pages, __ATOMIC_RELAXED);
gs->tier0_constant_pages = __atomic_load_n(&global_statistics.tier0_constant_pages, __ATOMIC_RELAXED);
for(size_t tier = 0; tier < storage_tiers ;tier++)
gs->db_points_stored_per_tier[tier] = __atomic_load_n(&global_statistics.db_points_stored_per_tier[tier], __ATOMIC_RELAXED);
@ -920,6 +941,38 @@ static void global_statistics_charts(void) {
rrdset_done(st_tier0_compression_info);
}
#endif
{
static RRDSET *st_t0_page_type = NULL;
static RRDDIM *rd_raw_pages = NULL;
static RRDDIM *rd_constant_pages = NULL;
if (unlikely(!st_t0_page_type)) {
st_t0_page_type = rrdset_create_localhost(
"netdata"
, "tier0_page_type"
, NULL
, "tier0_page_type"
, NULL
, "Tier 0 page type"
, "pages"
, "netdata"
, "stats"
, 131006
, localhost->rrd_update_every
, RRDSET_TYPE_LINE
);
rd_raw_pages = rrddim_add(st_t0_page_type, "raw", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rd_constant_pages = rrddim_add(st_t0_page_type, "constant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
}
rrddim_set_by_pointer(st_t0_page_type, rd_raw_pages, (collected_number) gs.tier0_raw_pages);
rrddim_set_by_pointer(st_t0_page_type, rd_constant_pages, (collected_number) gs.tier0_constant_pages);
rrdset_done(st_t0_page_type);
}
}
// ----------------------------------------------------------------------------

4
src/daemon/global_statistics.h
View File

@ -50,6 +50,10 @@ void global_statistics_gorilla_buffer_add_hot();
void global_statistics_tier0_disk_compressed_bytes(uint32_t size);
void global_statistics_tier0_disk_uncompressed_bytes(uint32_t size);
void global_statistic_raw_page_new();
void global_statistic_constant_page_new();
void global_statistic_constant_page_rm();
void global_statistics_web_request_completed(uint64_t dt,
uint64_t bytes_received,
uint64_t bytes_sent,

3
src/daemon/main.c
View File

@ -1172,6 +1172,9 @@ static void get_netdata_configured_variables() {
const char *page_type = config_get(CONFIG_SECTION_DB, "dbengine page type", "raw");
if (strcmp(page_type, "gorilla") == 0) {
tier_page_type[0] = PAGE_GORILLA_METRICS;
}
else if (strcmp(page_type, "constant") == 0) {
tier_page_type[0] = PAGE_CONSTANT_METRICS;
} else if (strcmp(page_type, "raw") != 0) {
netdata_log_error("Invalid dbengine page type ''%s' given. Defaulting to 'raw'.", page_type);
}

2
src/daemon/static_threads.c
View File

@ -61,7 +61,7 @@ const struct netdata_static_thread static_threads_common[] = {
.config_name = "netdata monitoring",
.env_name = "NETDATA_INTERNALS_MONITORING",
.global_variable = &global_statistics_enabled,
.enabled = 0,
.enabled = 1,
.thread = NULL,
.init_routine = NULL,
.start_routine = global_statistics_main

111
src/database/engine/page.c
View File

@ -2,6 +2,7 @@
#include "page.h"
#include "daemon/global_statistics.h"
#include "libnetdata/libnetdata.h"
typedef enum __attribute__((packed)) {
@ -20,6 +21,10 @@ typedef struct {
uint32_t size;
} page_raw_t;
typedef struct {
storage_number v;
uint32_t n;
} page_constant_t;
typedef struct {
size_t num_buffers;
@ -45,6 +50,7 @@ struct pgd {
union {
page_raw_t raw;
page_gorilla_t gorilla;
page_constant_t constant;
};
};
@ -165,7 +171,9 @@ PGD *pgd_create(uint8_t type, uint32_t slots)
pg->states = PGD_STATE_CREATED_FROM_COLLECTOR;
switch (type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
global_statistic_raw_page_new();
// intentional fall-through
case PAGE_TIER: {
uint32_t size = slots * page_type_size[type];
@ -176,6 +184,16 @@ PGD *pgd_create(uint8_t type, uint32_t slots)
pg->raw.data = pgd_data_aral_alloc(size);
break;
}
case PAGE_CONSTANT_METRICS: {
global_statistic_constant_page_new();
internal_fatal(slots == 1,
"DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type);
pg->constant.v = SN_EMPTY_SLOT;
pg->constant.n = 0;
break;
}
case PAGE_GORILLA_METRICS: {
internal_fatal(slots == 1,
"DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type);
@ -222,7 +240,7 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size)
switch (type)
{
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
pg->raw.size = size;
pg->used = size / page_type_size[type];
@ -231,6 +249,12 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size)
pg->raw.data = pgd_data_aral_alloc(size);
memcpy(pg->raw.data, base, size);
break;
case PAGE_CONSTANT_METRICS: {
memcpy(&pg->constant.v, base, sizeof(storage_number));
memcpy(&pg->constant.n, base + sizeof(storage_number), sizeof(uint32_t));
pg->used = pg->slots = pg->constant.n;
break;
}
case PAGE_GORILLA_METRICS:
internal_fatal(size == 0, "Asked to create page with 0 data!!!");
internal_fatal(size % sizeof(uint32_t), "Unaligned gorilla buffer size");
@ -268,10 +292,12 @@ void pgd_free(PGD *pg)
switch (pg->type)
{
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
pgd_data_aral_free(pg->raw.data, pg->raw.size);
break;
case PAGE_CONSTANT_METRICS:
break;
case PAGE_GORILLA_METRICS: {
if (pg->states & PGD_STATE_CREATED_FROM_DISK)
{
@ -365,10 +391,13 @@ uint32_t pgd_memory_footprint(PGD *pg)
size_t footprint = 0;
switch (pg->type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
footprint = sizeof(PGD) + pg->raw.size;
break;
case PAGE_CONSTANT_METRICS:
footprint = sizeof(PGD);
break;
case PAGE_GORILLA_METRICS: {
if (pg->states & PGD_STATE_CREATED_FROM_DISK)
footprint = sizeof(PGD) + pg->raw.size;
@ -393,7 +422,7 @@ uint32_t pgd_disk_footprint(PGD *pg)
size_t size = 0;
switch (pg->type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER: {
uint32_t used_size = pg->used * page_type_size[pg->type];
internal_fatal(used_size > pg->raw.size, "Wrong disk footprint page size");
@ -401,6 +430,10 @@ uint32_t pgd_disk_footprint(PGD *pg)
break;
}
case PAGE_CONSTANT_METRICS: {
size = sizeof(storage_number) + sizeof(uint32_t);
break;
}
case PAGE_GORILLA_METRICS: {
if (pg->states & PGD_STATE_CREATED_FROM_COLLECTOR ||
pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING ||
@ -443,11 +476,21 @@ void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size)
pgd_disk_footprint(pg), dst_size);
switch (pg->type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
memcpy(dst, pg->raw.data, dst_size);
break;
case PAGE_GORILLA_METRICS: {
case PAGE_CONSTANT_METRICS:
{
memcpy(dst, &pg->constant.v, sizeof(storage_number));
pg->constant.n = pg->used;
memcpy(dst + sizeof(storage_number), &pg->constant.n, sizeof(uint32_t));
break;
}
case PAGE_GORILLA_METRICS:
{
if ((pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) == 0)
fatal("Copying to extent is supported only for PGDs that are scheduled for flushing.");
@ -500,7 +543,7 @@ void pgd_append_point(PGD *pg,
fatal("Data collection on page already scheduled for flushing");
switch (pg->type) {
case PAGE_METRICS: {
case PAGE_RAW_METRICS: {
storage_number *tier0_metric_data = (storage_number *)pg->raw.data;
storage_number t = pack_storage_number(n, flags);
tier0_metric_data[pg->used++] = t;
@ -510,6 +553,43 @@ void pgd_append_point(PGD *pg,
break;
}
case PAGE_CONSTANT_METRICS: {
storage_number t = pack_storage_number(n, flags);
if (pg->used == 0)
{
pg->constant.v = t;
pg->used = 1;
}
else if (pg->constant.v == t)
{
pg->used++;
}
else
{
storage_number v = pg->constant.v;
global_statistic_constant_page_rm();
global_statistic_raw_page_new();
pg->type = PAGE_RAW_METRICS;
uint32_t size = pg->slots * page_type_size[pg->type];
pg->raw.size = size;
pg->raw.data = pgd_data_aral_alloc(size);
storage_number *data = (storage_number *) pg->raw.data;
for (uint32_t i = 0; i != pg->used; i++)
data[i] = v;
data[pg->used++] = t;
break;
}
if ((pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) && does_storage_number_exist(t))
pg->options &= ~PAGE_OPTION_ALL_VALUES_EMPTY;
break;
}
case PAGE_TIER: {
storage_number_tier1_t *tier12_metric_data = (storage_number_tier1_t *)pg->raw.data;
storage_number_tier1_t t;
@ -560,8 +640,9 @@ static void pgdc_seek(PGDC *pgdc, uint32_t position)
PGD *pg = pgdc->pgd;
switch (pg->type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
case PAGE_CONSTANT_METRICS:
pgdc->slots = pgdc->pgd->used;
break;
case PAGE_GORILLA_METRICS: {
@ -634,7 +715,7 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
switch (pgdc->pgd->type)
{
case PAGE_METRICS: {
case PAGE_RAW_METRICS: {
storage_number *array = (storage_number *) pgdc->pgd->raw.data;
storage_number n = array[pgdc->position++];
@ -645,6 +726,16 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
return true;
}
case PAGE_CONSTANT_METRICS: {
storage_number n = pgdc->pgd->constant.v;
sp->min = sp->max = sp->sum = unpack_storage_number(n);
sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS);
sp->count = 1;
sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0;
return true;
}
case PAGE_TIER: {
storage_number_tier1_t *array = (storage_number_tier1_t *) pgdc->pgd->raw.data;
storage_number_tier1_t n = array[pgdc->position++];

10
src/database/engine/page_test.cc
View File

@ -38,7 +38,7 @@ static uint8_t page_type = PAGE_GORILLA_METRICS;
static size_t slots_for_page(size_t n) {
switch (page_type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
return 1024;
case PAGE_GORILLA_METRICS:
return n;
@ -200,7 +200,7 @@ TEST(PGD, MemoryFootprint) {
uint32_t footprint = 0;
switch (pgd_type(pg)) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
footprint = slots * sizeof(uint32_t);
break;
case PAGE_GORILLA_METRICS:
@ -225,7 +225,7 @@ TEST(PGD, MemoryFootprint) {
uint32_t abs_error = 0;
switch (pgd_type(pg)) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
abs_error = 128;
break;
case PAGE_GORILLA_METRICS:
@ -256,7 +256,7 @@ TEST(PGD, DiskFootprint) {
uint32_t footprint = 0;
switch (pgd_type(pg)) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
footprint = used_slots * sizeof(uint32_t);
break;
case PAGE_GORILLA_METRICS:
@ -279,7 +279,7 @@ TEST(PGD, DiskFootprint) {
}
switch (pgd_type(pg)) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
footprint = used_slots * sizeof(uint32_t);
break;
case PAGE_GORILLA_METRICS:

17
src/database/engine/pdc.c
View File

@ -635,7 +635,7 @@ inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_
size_t entries = 0;
switch (descr->type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
end_time_s = descr->end_time_ut / USEC_PER_SEC;
entries = 0;
@ -644,6 +644,10 @@ inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_
end_time_s = start_time_s + descr->gorilla.delta_time_s;
entries = descr->gorilla.entries;
break;
case PAGE_CONSTANT_METRICS:
end_time_s = start_time_s + descr->constant.delta_time_s;
entries = descr->constant.entries;
break;
default:
// Nothing to do. Validate page will notify the user.
break;
@ -689,7 +693,7 @@ VALIDATED_PAGE_DESCRIPTOR validate_page(
bool known_page_type = true;
switch (page_type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
// always calculate entries by size
vd.entries = page_entries_by_size(vd.page_length, vd.point_size);
@ -702,6 +706,10 @@ VALIDATED_PAGE_DESCRIPTOR validate_page(
internal_fatal(entries == 0, "0 number of entries found on gorilla page");
vd.entries = entries;
break;
case PAGE_CONSTANT_METRICS:
internal_fatal(entries == 0, "0 number of entries found on constant page");
vd.entries = entries;
break;
default:
known_page_type = false;
break;
@ -873,13 +881,16 @@ static void epdl_extent_loading_error_log(struct rrdengine_instance *ctx, EPDL *
if (descr) {
start_time_s = (time_t)(descr->start_time_ut / USEC_PER_SEC);
switch (descr->type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
end_time_s = (time_t)(descr->end_time_ut / USEC_PER_SEC);
break;
case PAGE_GORILLA_METRICS:
end_time_s = (time_t) start_time_s + (descr->gorilla.delta_time_s);
break;
case PAGE_CONSTANT_METRICS:
end_time_s = (time_t) start_time_s + (descr->constant.delta_time_s);
break;
}
uuid_unparse_lower(descr->uuid, uuid);
used_descr = true;

12
src/database/engine/rrddiskprotocol.h
View File

@ -36,10 +36,11 @@ struct rrdeng_df_sb {
/*
* Page types
*/
#define PAGE_METRICS (0)
#define PAGE_RAW_METRICS (0)
#define PAGE_TIER (1)
#define PAGE_GORILLA_METRICS (2)
#define PAGE_TYPE_MAX 2 // Maximum page type (inclusive)
#define PAGE_CONSTANT_METRICS (3)
#define PAGE_TYPE_MAX 3 // Maximum page type (inclusive)
/*
* Data file page descriptor
@ -56,6 +57,11 @@ struct rrdeng_extent_page_descr {
uint32_t delta_time_s;
} gorilla __attribute__((packed));
struct {
uint32_t entries;
uint32_t delta_time_s;
} constant __attribute__((packed));
uint64_t end_time_ut;
};
} __attribute__ ((packed));
@ -127,4 +133,4 @@ struct rrdeng_jf_store_data {
struct rrdeng_extent_page_descr descr[];
} __attribute__ ((packed));
#endif /* NETDATA_RRDDISKPROTOCOL_H */
#endif /* NETDATA_RRDDISKPROTOCOL_H */

6
src/database/engine/rrdengine.c
View File

@ -844,7 +844,7 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta
header->descr[i].start_time_ut = descr->start_time_ut;
switch (descr->type) {
case PAGE_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
header->descr[i].end_time_ut = descr->end_time_ut;
break;
@ -852,6 +852,10 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta
header->descr[i].gorilla.delta_time_s = (uint32_t) ((descr->end_time_ut - descr->start_time_ut) / USEC_PER_SEC);
header->descr[i].gorilla.entries = pgd_slots_used(descr->pgd);
break;
case PAGE_CONSTANT_METRICS:
header->descr[i].constant.delta_time_s = (uint32_t) ((descr->end_time_ut - descr->start_time_ut) / USEC_PER_SEC);
header->descr[i].constant.entries = pgd_slots_used(descr->pgd);
break;
default:
fatal("Unknown page type: %uc", descr->type);
}

14
src/database/engine/rrdengineapi.c
View File

@ -16,7 +16,7 @@ struct rrdengine_instance multidb_ctx_storage_tier4;
#error RRD_STORAGE_TIERS is not 5 - you need to add allocations here
#endif
struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
uint8_t tier_page_type[RRD_STORAGE_TIERS] = {PAGE_METRICS, PAGE_TIER, PAGE_TIER, PAGE_TIER, PAGE_TIER};
uint8_t tier_page_type[RRD_STORAGE_TIERS] = {PAGE_CONSTANT_METRICS, PAGE_TIER, PAGE_TIER, PAGE_TIER, PAGE_TIER};
#if defined(ENV32BIT)
size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192};
@ -24,14 +24,15 @@ size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192};
size_t tier_page_size[RRD_STORAGE_TIERS] = {4096, 2048, 384, 384, 384};
#endif
#if PAGE_TYPE_MAX != 2
#error PAGE_TYPE_MAX is not 2 - you need to add allocations here
#if PAGE_TYPE_MAX != 3
#error PAGE_TYPE_MAX is not 3 - you need to add allocations here
#endif
size_t page_type_size[256] = {
[PAGE_METRICS] = sizeof(storage_number),
[PAGE_RAW_METRICS] = sizeof(storage_number),
[PAGE_TIER] = sizeof(storage_number_tier1_t),
[PAGE_GORILLA_METRICS] = sizeof(storage_number)
[PAGE_GORILLA_METRICS] = sizeof(storage_number),
[PAGE_CONSTANT_METRICS] = sizeof(storage_number),
};
__attribute__((constructor)) void initialize_multidb_ctx(void) {
@ -457,7 +458,8 @@ static PGD *rrdeng_alloc_new_page_data(struct rrdeng_collect_handle *handle, siz
*data_size = size;
switch (ctx->config.page_type) {
case PAGE_METRICS:
case PAGE_CONSTANT_METRICS:
case PAGE_RAW_METRICS:
case PAGE_TIER:
d = pgd_create(ctx->config.page_type, slots);
break;