diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index 213ef57991..c05ddd17d7 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -627,6 +627,33 @@ PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_ return PARSER_RC_OK; } +PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION *plugins_action) +{ + char *machine_guid = words[1]; + char *hostname = words[2]; + char *registry_hostname = words[3]; + char *update_every_s = words[4]; + char *os = words[5]; + char *timezone = words[6]; + char *tags = words[7]; + + int update_every = 1; + if (likely(update_every_s && *update_every_s)) + update_every = str2i(update_every_s); + if (unlikely(!update_every)) + update_every = 1; + + debug(D_PLUGINSD, "HOST PARSED: guid=%s, hostname=%s, reg_host=%s, update=%d, os=%s, timezone=%s, tags=%s", + machine_guid, hostname, registry_hostname, update_every, os, timezone, tags); + + if (plugins_action->host_action) { + return plugins_action->host_action( + user, machine_guid, hostname, registry_hostname, update_every, os, timezone, tags); + } + + return PARSER_RC_OK; +} + // New plugins.d parser inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations) diff --git a/database/engine/global_uuid_map/global_uuid_map.c b/database/engine/global_uuid_map/global_uuid_map.c index f55ea5d7ad..a2fedfc568 100644 --- a/database/engine/global_uuid_map/global_uuid_map.c +++ b/database/engine/global_uuid_map/global_uuid_map.c @@ -155,6 +155,7 @@ GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes) return GUID_TYPE_NOSPACE; strncpyz(object, (char *) *PValue+1, max_bytes - 1); break; + case GUID_TYPE_HOST: case GUID_TYPE_CHART: case GUID_TYPE_DIMENSION: if (unlikely(max_bytes < (size_t) value_type * 16)) @@ -221,6 +222,11 @@ int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16); memcpy(target_object + 17, temp_uuid, 16); break; + case GUID_TYPE_HOST: + target_object = mallocz(17); + target_object[0] = object_type; + memcpy(target_object + 1, (((RRDHOST *)object))->host_uuid, 16); + break; case GUID_TYPE_CHAR: target_object = mallocz(strlen((char *) object)+2); target_object[0] = object_type; diff --git a/database/engine/metadata_log/compaction.c b/database/engine/metadata_log/compaction.c index 150d124ba9..8bdfc47049 100644 --- a/database/engine/metadata_log/compaction.c +++ b/database/engine/metadata_log/compaction.c @@ -79,10 +79,10 @@ static void compaction_test_quota(struct metalog_worker_config *wc) static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid) { GUID_TYPE ret; - RRDHOST *host = ctx->rrdeng_ctx->host; RRDSET *st; RRDDIM *rd; BUFFER *buffer; + RRDHOST *host = ctx->rrdeng_ctx->host; ret = find_object_by_guid(uuid, NULL, 0); switch (ret) { @@ -92,6 +92,11 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid) case GUID_TYPE_CHART: st = metalog_get_chart_from_uuid(ctx, uuid); if (st) { + if (ctx->current_compaction_id > st->rrdhost->compaction_id) { + error("Forcing compaction of HOST %s from CHART %s", st->rrdhost->hostname, st->id); + compact_record_by_uuid(ctx, &st->rrdhost->host_uuid); + } + if (ctx->current_compaction_id > st->compaction_id) { st->compaction_id = ctx->current_compaction_id; buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id); @@ -106,11 +111,13 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid) case GUID_TYPE_DIMENSION: rd = metalog_get_dimension_from_uuid(ctx, uuid); if (rd) { + if (ctx->current_compaction_id > rd->rrdset->rrdhost->compaction_id) { + error("Forcing compaction of HOST %s", rd->rrdset->rrdhost->hostname); + compact_record_by_uuid(ctx, &rd->rrdset->rrdhost->host_uuid); + } if (ctx->current_compaction_id > rd->rrdset->compaction_id) { - error("Forcing compaction of chart %s", rd->rrdset->id); - rd->rrdset->compaction_id = ctx->current_compaction_id; - buffer = metalog_update_chart_buffer(rd->rrdset, ctx->current_compaction_id); - metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, rd->rrdset->chart_uuid, 1); + error("Forcing compaction of CHART %s", rd->rrdset->id); + compact_record_by_uuid(ctx, rd->rrdset->chart_uuid); } else if (ctx->current_compaction_id > rd->state->compaction_id) { rd->state->compaction_id = ctx->current_compaction_id; buffer = metalog_update_dimension_buffer(rd); @@ -123,6 +130,8 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid) } break; case GUID_TYPE_HOST: + //TODO: will be enabled when multidb is activated + //RRDHOST *host = metalog_get_host_from_uuid(ctx, uuid); if (ctx->current_compaction_id > host->compaction_id) { host->compaction_id = ctx->current_compaction_id; buffer = metalog_update_host_buffer(host); diff --git a/database/engine/metadata_log/logfile.c b/database/engine/metadata_log/logfile.c index 2eb9971cd3..2a025b16bb 100644 --- a/database/engine/metadata_log/logfile.c +++ b/database/engine/metadata_log/logfile.c @@ -675,6 +675,7 @@ static int scan_metalog_files(struct metalog_instance *ctx) failed_to_load = matched_files; goto after_failed_to_parse; } + parser_add_keyword(parser, PLUGINSD_KEYWORD_HOST, metalog_pluginsd_host); parser_add_keyword(parser, PLUGINSD_KEYWORD_GUID, pluginsd_guid); parser_add_keyword(parser, PLUGINSD_KEYWORD_CONTEXT, pluginsd_context); parser_add_keyword(parser, PLUGINSD_KEYWORD_TOMBSTONE, pluginsd_tombstone); @@ -683,6 +684,8 @@ static int scan_metalog_files(struct metalog_instance *ctx) parser->plugins_action->guid_action = &metalog_pluginsd_guid_action; parser->plugins_action->context_action = &metalog_pluginsd_context_action; parser->plugins_action->tombstone_action = &metalog_pluginsd_tombstone_action; + parser->plugins_action->host_action = &metalog_pluginsd_host_action; + metalog_parser_object.parser = parser; ctx->metalog_parser_object = &metalog_parser_object; diff --git a/database/engine/metadata_log/metadatalogapi.c b/database/engine/metadata_log/metadatalogapi.c index b4266675d5..7f45a44db4 100755 --- a/database/engine/metadata_log/metadatalogapi.c +++ b/database/engine/metadata_log/metadatalogapi.c @@ -266,6 +266,22 @@ void metalog_commit_delete_dimension(RRDDIM *rd) metalog_commit_deletion_record(ctx, buffer); } +RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *host_guid) +{ + UNUSED(ctx); + GUID_TYPE ret; + char machine_guid[37]; + + uuid_unparse_lower(*host_guid, machine_guid); + ret = find_object_by_guid(host_guid, NULL, 0); + if (unlikely(GUID_TYPE_HOST != ret)) { + error("Host with GUID %s not found in the global map", machine_guid); + return NULL; + } + RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0); + return host; +} + RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid) { GUID_TYPE ret; diff --git a/database/engine/metadata_log/metadatalogapi.h b/database/engine/metadata_log/metadatalogapi.h index d9dd589848..3808ab6ce3 100644 --- a/database/engine/metadata_log/metadatalogapi.h +++ b/database/engine/metadata_log/metadatalogapi.h @@ -16,6 +16,7 @@ extern void metalog_commit_delete_dimension(RRDDIM *rd); extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid); extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid); +extern RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *uuid); extern void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid); /* must call once before using anything */ diff --git a/database/engine/metadata_log/metalogpluginsd.c b/database/engine/metadata_log/metalogpluginsd.c index 789ad47192..4e8ba63628 100755 --- a/database/engine/metadata_log/metalogpluginsd.c +++ b/database/engine/metadata_log/metalogpluginsd.c @@ -4,6 +4,63 @@ #include "metadatalog.h" #include "metalogpluginsd.h" +PARSER_RC metalog_pluginsd_host_action( + void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone, + char *tags) +{ + struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private; + + RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0); + if (host) + goto write_replay; + + if (strcmp(machine_guid, registry_get_this_machine_guid()) == 0) { + struct metalog_record record; + struct metadata_logfile *metalogfile = state->metalogfile; + + uuid_parse(machine_guid, record.uuid); + mlf_record_insert(metalogfile, &record); + return PARSER_RC_OK; + } + + // Ignore HOST command for now + // TODO: Remove when the next task is completed ie. accept new children in the lcoalhost / multidb + return PARSER_RC_OK; + + host = rrdhost_create( + hostname + , registry_hostname + , machine_guid + , os + , timezone + , tags + , NULL + , NULL + , update_every + , 3600 + , RRD_MEMORY_MODE_DBENGINE + , 0 // health enabled + , 0 // Push enabled + , NULL + , NULL + , NULL + , callocz(1, sizeof(struct rrdhost_system_info)) + , 0 // localhost + , 1 // archived + ); + +write_replay: + if (host) { /* It's a valid object */ + struct metalog_record record; + struct metadata_logfile *metalogfile = state->metalogfile; + + uuid_copy(record.uuid, host->host_uuid); + mlf_record_insert(metalogfile, &record); + } + + return PARSER_RC_OK; +} + PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin, char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options) diff --git a/database/engine/metadata_log/metalogpluginsd.h b/database/engine/metadata_log/metalogpluginsd.h index b29a9db33a..ad618d42c2 100644 --- a/database/engine/metadata_log/metalogpluginsd.h +++ b/database/engine/metadata_log/metalogpluginsd.h @@ -25,5 +25,7 @@ extern PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char extern PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid); extern PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid); extern PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid); +extern PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION *plugins_action); +extern PARSER_RC metalog_pluginsd_host_action(void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone, char *tags); #endif /* NETDATA_METALOGPLUGINSD_H */ diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index 315d4ad51d..a309b85f60 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -916,11 +916,16 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p if (ctx->worker_config.error) { goto error_after_rrdeng_worker; } - error = metalog_init(ctx); - if(error) { - error("Failed to initialize metadata log file event loop."); - goto error_after_rrdeng_worker; + if ((strcmp(host->machine_guid, registry_get_this_machine_guid()) == 0) || (!rrdhost_flag_check(host, RRDHOST_FLAG_MULTIHOST))) { + info("Metadatalog init for host %s starting...", host->hostname); + error = metalog_init(ctx); + if (error) { + error("Failed to initialize metadata log file event loop."); + goto error_after_rrdeng_worker; + } } + else + info("No metadatalog init for host %s", host->hostname); return 0; error_after_rrdeng_worker: diff --git a/database/rrd.h b/database/rrd.h index b6d00ebcce..b01cd28a5e 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -560,6 +560,8 @@ typedef enum rrdhost_flags { RRDHOST_FLAG_DELETE_ORPHAN_HOST = 1 << 2, // delete the entire host when orphan RRDHOST_FLAG_BACKEND_SEND = 1 << 3, // send it to backends RRDHOST_FLAG_BACKEND_DONT_SEND = 1 << 4, // don't send it to backends + RRDHOST_FLAG_ARCHIVED = 1 << 5, // The host is archived, no collected charts yet + RRDHOST_FLAG_MULTIHOST = 1 << 6, // Host belongs to localhost/megadb } RRDHOST_FLAGS; #ifdef HAVE_C___ATOMIC @@ -1195,6 +1197,12 @@ extern void rrdset_delete_custom(RRDSET *st, int db_rotated); extern void rrdset_delete_obsolete_dimensions(RRDSET *st); extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host); +extern RRDHOST *rrdhost_create( + const char *hostname, const char *registry_hostname, const char *guid, const char *os, const char *timezone, + const char *tags, const char *program_name, const char *program_version, int update_every, long entries, + RRD_MEMORY_MODE memory_mode, unsigned int health_enabled, unsigned int rrdpush_enabled, char *rrdpush_destination, + char *rrdpush_api_key, char *rrdpush_send_charts_matching, struct rrdhost_system_info *system_info, + int is_localhost, int is_archived); #endif /* NETDATA_RRD_INTERNALS */ diff --git a/database/rrdhost.c b/database/rrdhost.c index 4e8488534f..66996a86bc 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -123,7 +123,8 @@ RRDHOST *rrdhost_create(const char *hostname, char *rrdpush_api_key, char *rrdpush_send_charts_matching, struct rrdhost_system_info *system_info, - int is_localhost + int is_localhost, + int is_archived ) { debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid); @@ -182,6 +183,10 @@ RRDHOST *rrdhost_create(const char *hostname, avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare); avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare); + if (is_archived) { + rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); + info("Host %s is created in archived mode", hostname); + } if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1)) rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); @@ -276,6 +281,10 @@ RRDHOST *rrdhost_create(const char *hostname, if (unlikely(-1 == uuid_parse(host->machine_guid, host->host_uuid))) { error("Host machine GUID is not valid."); } + if (unlikely(find_or_generate_guid((void *) host, &host->host_uuid, GUID_TYPE_HOST, 1))) + error("Failed to store machine GUID to global map"); + else + info("Added %s to global map for host %s", host->machine_guid, host->hostname); host->objects_nr = 1; host->compaction_id = 0; char dbenginepath[FILENAME_MAX + 1]; @@ -367,7 +376,7 @@ RRDHOST *rrdhost_create(const char *hostname, rrd_hosts_available++; #ifdef ENABLE_DBENGINE - if (likely(!is_localhost && host && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)) + if (likely(!is_localhost && !is_archived && host && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)) metalog_commit_update_host(host); #endif return host; @@ -494,6 +503,7 @@ RRDHOST *rrdhost_find_or_create( , rrdpush_send_charts_matching , system_info , 0 + , 0 ); } else { @@ -590,6 +600,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { , default_rrdpush_send_charts_matching , system_info , 1 + , 0 ); rrd_unlock(); web_client_api_v1_management_init(); diff --git a/database/rrdset.c b/database/rrdset.c index 0483c5ac84..05dbed52e8 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -543,6 +543,10 @@ RRDSET *rrdset_create_custom( if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) { rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED); changed_from_archived_to_active = 1; + if (rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_ARCHIVED)) { + rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED); + info("Host %s is not in archived mode anymore", st->rrdhost->hostname); + } mark_rebuild |= META_CHART_ACTIVATED; } char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_family = NULL, *old_context = NULL, @@ -683,6 +687,10 @@ RRDSET *rrdset_create_custom( rrdhost_unlock(host); rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); + if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) { + rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED); + rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED); + } return st; } diff --git a/parser/parser.h b/parser/parser.h index 64127a71d1..86a837e803 100644 --- a/parser/parser.h +++ b/parser/parser.h @@ -35,6 +35,8 @@ typedef struct pluginsd_action { PARSER_RC (*guid_action)(void *user, uuid_t *uuid); PARSER_RC (*context_action)(void *user, uuid_t *uuid); PARSER_RC (*tombstone_action)(void *user, uuid_t *uuid); + PARSER_RC (*host_action)(void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, + char *timezone, char *tags); } PLUGINSD_ACTION; typedef enum parser_input_type {