opensourcemirror
/
netdata
mirror of https://github.com/netdata/netdata.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Implemented the HOST command in metadata log replay (#9489)

This commit is contained in:
Stelios Fragkakis 2020-07-17 00:00:00 +03:00 committed by GitHub
parent 36f0ad220e
commit 5871b37cf6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 166 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
collectors/plugins.d/pluginsd_parser.c
View File

@ -627,6 +627,33 @@ PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_
return PARSER_RC_OK;
}
PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION *plugins_action)
{
char *machine_guid = words[1];
char *hostname = words[2];
char *registry_hostname = words[3];
char *update_every_s = words[4];
char *os = words[5];
char *timezone = words[6];
char *tags = words[7];
int update_every = 1;
if (likely(update_every_s && *update_every_s))
update_every = str2i(update_every_s);
if (unlikely(!update_every))
update_every = 1;
debug(D_PLUGINSD, "HOST PARSED: guid=%s, hostname=%s, reg_host=%s, update=%d, os=%s, timezone=%s, tags=%s",
machine_guid, hostname, registry_hostname, update_every, os, timezone, tags);
if (plugins_action->host_action) {
return plugins_action->host_action(
user, machine_guid, hostname, registry_hostname, update_every, os, timezone, tags);
}
return PARSER_RC_OK;
}
// New plugins.d parser
inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations)

6
database/engine/global_uuid_map/global_uuid_map.c
View File

@ -155,6 +155,7 @@ GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes)
return GUID_TYPE_NOSPACE;
strncpyz(object, (char *) *PValue+1, max_bytes - 1);
break;
case GUID_TYPE_HOST:
case GUID_TYPE_CHART:
case GUID_TYPE_DIMENSION:
if (unlikely(max_bytes < (size_t) value_type * 16))
@ -221,6 +222,11 @@ int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int
memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16);
memcpy(target_object + 17, temp_uuid, 16);
break;
case GUID_TYPE_HOST:
target_object = mallocz(17);
target_object[0] = object_type;
memcpy(target_object + 1, (((RRDHOST *)object))->host_uuid, 16);
break;
case GUID_TYPE_CHAR:
target_object = mallocz(strlen((char *) object)+2);
target_object[0] = object_type;

19
database/engine/metadata_log/compaction.c
View File

@ -79,10 +79,10 @@ static void compaction_test_quota(struct metalog_worker_config *wc)
static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
{
GUID_TYPE ret;
RRDHOST *host = ctx->rrdeng_ctx->host;
RRDSET *st;
RRDDIM *rd;
BUFFER *buffer;
RRDHOST *host = ctx->rrdeng_ctx->host;
ret = find_object_by_guid(uuid, NULL, 0);
switch (ret) {
@ -92,6 +92,11 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
case GUID_TYPE_CHART:
st = metalog_get_chart_from_uuid(ctx, uuid);
if (st) {
if (ctx->current_compaction_id > st->rrdhost->compaction_id) {
error("Forcing compaction of HOST %s from CHART %s", st->rrdhost->hostname, st->id);
compact_record_by_uuid(ctx, &st->rrdhost->host_uuid);
}
if (ctx->current_compaction_id > st->compaction_id) {
st->compaction_id = ctx->current_compaction_id;
buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id);
@ -106,11 +111,13 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
case GUID_TYPE_DIMENSION:
rd = metalog_get_dimension_from_uuid(ctx, uuid);
if (rd) {
if (ctx->current_compaction_id > rd->rrdset->rrdhost->compaction_id) {
error("Forcing compaction of HOST %s", rd->rrdset->rrdhost->hostname);
compact_record_by_uuid(ctx, &rd->rrdset->rrdhost->host_uuid);
}
if (ctx->current_compaction_id > rd->rrdset->compaction_id) {
error("Forcing compaction of chart %s", rd->rrdset->id);
rd->rrdset->compaction_id = ctx->current_compaction_id;
buffer = metalog_update_chart_buffer(rd->rrdset, ctx->current_compaction_id);
metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, rd->rrdset->chart_uuid, 1);
error("Forcing compaction of CHART %s", rd->rrdset->id);
compact_record_by_uuid(ctx, rd->rrdset->chart_uuid);
} else if (ctx->current_compaction_id > rd->state->compaction_id) {
rd->state->compaction_id = ctx->current_compaction_id;
buffer = metalog_update_dimension_buffer(rd);
@ -123,6 +130,8 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
}
break;
case GUID_TYPE_HOST:
//TODO: will be enabled when multidb is activated
//RRDHOST *host = metalog_get_host_from_uuid(ctx, uuid);
if (ctx->current_compaction_id > host->compaction_id) {
host->compaction_id = ctx->current_compaction_id;
buffer = metalog_update_host_buffer(host);

3
database/engine/metadata_log/logfile.c
View File

@ -675,6 +675,7 @@ static int scan_metalog_files(struct metalog_instance *ctx)
failed_to_load = matched_files;
goto after_failed_to_parse;
}
parser_add_keyword(parser, PLUGINSD_KEYWORD_HOST, metalog_pluginsd_host);
parser_add_keyword(parser, PLUGINSD_KEYWORD_GUID, pluginsd_guid);
parser_add_keyword(parser, PLUGINSD_KEYWORD_CONTEXT, pluginsd_context);
parser_add_keyword(parser, PLUGINSD_KEYWORD_TOMBSTONE, pluginsd_tombstone);
@ -683,6 +684,8 @@ static int scan_metalog_files(struct metalog_instance *ctx)
parser->plugins_action->guid_action = &metalog_pluginsd_guid_action;
parser->plugins_action->context_action = &metalog_pluginsd_context_action;
parser->plugins_action->tombstone_action = &metalog_pluginsd_tombstone_action;
parser->plugins_action->host_action = &metalog_pluginsd_host_action;
metalog_parser_object.parser = parser;
ctx->metalog_parser_object = &metalog_parser_object;

16
database/engine/metadata_log/metadatalogapi.c
View File

@ -266,6 +266,22 @@ void metalog_commit_delete_dimension(RRDDIM *rd)
metalog_commit_deletion_record(ctx, buffer);
}
RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *host_guid)
{
UNUSED(ctx);
GUID_TYPE ret;
char machine_guid[37];
uuid_unparse_lower(*host_guid, machine_guid);
ret = find_object_by_guid(host_guid, NULL, 0);
if (unlikely(GUID_TYPE_HOST != ret)) {
error("Host with GUID %s not found in the global map", machine_guid);
return NULL;
}
RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
return host;
}
RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid)
{
GUID_TYPE ret;

1
database/engine/metadata_log/metadatalogapi.h
View File

@ -16,6 +16,7 @@ extern void metalog_commit_delete_dimension(RRDDIM *rd);
extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid);
extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
extern RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *uuid);
extern void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
/* must call once before using anything */

57
database/engine/metadata_log/metalogpluginsd.c
View File

@ -4,6 +4,63 @@
#include "metadatalog.h"
#include "metalogpluginsd.h"
PARSER_RC metalog_pluginsd_host_action(
void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone,
char *tags)
{
struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
if (host)
goto write_replay;
if (strcmp(machine_guid, registry_get_this_machine_guid()) == 0) {
struct metalog_record record;
struct metadata_logfile *metalogfile = state->metalogfile;
uuid_parse(machine_guid, record.uuid);
mlf_record_insert(metalogfile, &record);
return PARSER_RC_OK;
}
// Ignore HOST command for now
// TODO: Remove when the next task is completed ie. accept new children in the lcoalhost / multidb
return PARSER_RC_OK;
host = rrdhost_create(
hostname
, registry_hostname
, machine_guid
, os
, timezone
, tags
, NULL
, NULL
, update_every
, 3600
, RRD_MEMORY_MODE_DBENGINE
, 0 // health enabled
, 0 // Push enabled
, NULL
, NULL
, NULL
, callocz(1, sizeof(struct rrdhost_system_info))
, 0 // localhost
, 1 // archived
);
write_replay:
if (host) { /* It's a valid object */
struct metalog_record record;
struct metadata_logfile *metalogfile = state->metalogfile;
uuid_copy(record.uuid, host->host_uuid);
mlf_record_insert(metalogfile, &record);
}
return PARSER_RC_OK;
}
PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
char *title, char *units, char *plugin, char *module, int priority,
int update_every, RRDSET_TYPE chart_type, char *options)

2
database/engine/metadata_log/metalogpluginsd.h
View File

@ -25,5 +25,7 @@ extern PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char
extern PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid);
extern PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid);
extern PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid);
extern PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION *plugins_action);
extern PARSER_RC metalog_pluginsd_host_action(void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone, char *tags);
#endif /* NETDATA_METALOGPLUGINSD_H */

13
database/engine/rrdengineapi.c
View File

@ -916,11 +916,16 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
if (ctx->worker_config.error) {
goto error_after_rrdeng_worker;
}
error = metalog_init(ctx);
if(error) {
error("Failed to initialize metadata log file event loop.");
goto error_after_rrdeng_worker;
if ((strcmp(host->machine_guid, registry_get_this_machine_guid()) == 0) || (!rrdhost_flag_check(host, RRDHOST_FLAG_MULTIHOST))) {
info("Metadatalog init for host %s starting...", host->hostname);
error = metalog_init(ctx);
if (error) {
error("Failed to initialize metadata log file event loop.");
goto error_after_rrdeng_worker;
}
}
else
info("No metadatalog init for host %s", host->hostname);
return 0;
error_after_rrdeng_worker:

8
database/rrd.h
View File

@ -560,6 +560,8 @@ typedef enum rrdhost_flags {
RRDHOST_FLAG_DELETE_ORPHAN_HOST = 1 << 2, // delete the entire host when orphan
RRDHOST_FLAG_BACKEND_SEND = 1 << 3, // send it to backends
RRDHOST_FLAG_BACKEND_DONT_SEND = 1 << 4, // don't send it to backends
RRDHOST_FLAG_ARCHIVED = 1 << 5, // The host is archived, no collected charts yet
RRDHOST_FLAG_MULTIHOST = 1 << 6, // Host belongs to localhost/megadb
} RRDHOST_FLAGS;
#ifdef HAVE_C___ATOMIC
@ -1195,6 +1197,12 @@ extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host);
extern RRDHOST *rrdhost_create(
const char *hostname, const char *registry_hostname, const char *guid, const char *os, const char *timezone,
const char *tags, const char *program_name, const char *program_version, int update_every, long entries,
RRD_MEMORY_MODE memory_mode, unsigned int health_enabled, unsigned int rrdpush_enabled, char *rrdpush_destination,
char *rrdpush_api_key, char *rrdpush_send_charts_matching, struct rrdhost_system_info *system_info,
int is_localhost, int is_archived);
#endif /* NETDATA_RRD_INTERNALS */

15
database/rrdhost.c
View File

@ -123,7 +123,8 @@ RRDHOST *rrdhost_create(const char *hostname,
char *rrdpush_api_key,
char *rrdpush_send_charts_matching,
struct rrdhost_system_info *system_info,
int is_localhost
int is_localhost,
int is_archived
) {
debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
@ -182,6 +183,10 @@ RRDHOST *rrdhost_create(const char *hostname,
avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare);
avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare);
if (is_archived) {
rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
info("Host %s is created in archived mode", hostname);
}
if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1))
rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS);
@ -276,6 +281,10 @@ RRDHOST *rrdhost_create(const char *hostname,
if (unlikely(-1 == uuid_parse(host->machine_guid, host->host_uuid))) {
error("Host machine GUID is not valid.");
}
if (unlikely(find_or_generate_guid((void *) host, &host->host_uuid, GUID_TYPE_HOST, 1)))
error("Failed to store machine GUID to global map");
else
info("Added %s to global map for host %s", host->machine_guid, host->hostname);
host->objects_nr = 1;
host->compaction_id = 0;
char dbenginepath[FILENAME_MAX + 1];
@ -367,7 +376,7 @@ RRDHOST *rrdhost_create(const char *hostname,
rrd_hosts_available++;
#ifdef ENABLE_DBENGINE
if (likely(!is_localhost && host && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
if (likely(!is_localhost && !is_archived && host && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
metalog_commit_update_host(host);
#endif
return host;
@ -494,6 +503,7 @@ RRDHOST *rrdhost_find_or_create(
, rrdpush_send_charts_matching
, system_info
, 0
, 0
);
}
else {
@ -590,6 +600,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
, default_rrdpush_send_charts_matching
, system_info
, 1
, 0
);
rrd_unlock();
web_client_api_v1_management_init();

8
database/rrdset.c
View File

@ -543,6 +543,10 @@ RRDSET *rrdset_create_custom(
if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
changed_from_archived_to_active = 1;
if (rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_ARCHIVED)) {
rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED);
info("Host %s is not in archived mode anymore", st->rrdhost->hostname);
}
mark_rebuild |= META_CHART_ACTIVATED;
}
char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_family = NULL, *old_context = NULL,
@ -683,6 +687,10 @@ RRDSET *rrdset_create_custom(
rrdhost_unlock(host);
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED);
}
return st;
}

2
parser/parser.h
View File

@ -35,6 +35,8 @@ typedef struct pluginsd_action {
PARSER_RC (*guid_action)(void *user, uuid_t *uuid);
PARSER_RC (*context_action)(void *user, uuid_t *uuid);
PARSER_RC (*tombstone_action)(void *user, uuid_t *uuid);
PARSER_RC (*host_action)(void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os,
char *timezone, char *tags);
} PLUGINSD_ACTION;
typedef enum parser_input_type {