sqlite3 global statistics (#13594)

This commit is contained in:
Costa Tsaousis 2022-08-31 10:04:14 +03:00 committed by GitHub
parent c9620ca3ed
commit 77b0e7bccd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 267 additions and 95 deletions

View File

@ -30,6 +30,14 @@ static struct global_statistics {
volatile uint64_t rrdr_queries_made;
volatile uint64_t rrdr_db_points_read;
volatile uint64_t rrdr_result_points_generated;
volatile uint64_t sqlite3_queries_made;
volatile uint64_t sqlite3_queries_ok;
volatile uint64_t sqlite3_queries_failed;
volatile uint64_t sqlite3_queries_failed_busy;
volatile uint64_t sqlite3_queries_failed_locked;
volatile uint64_t sqlite3_rows;
} global_statistics = {
.connected_clients = 0,
.web_requests = 0,
@ -45,6 +53,27 @@ static struct global_statistics {
.rrdr_result_points_generated = 0,
};
void sqlite3_query_completed(bool success, bool busy, bool locked) {
__atomic_fetch_add(&global_statistics.sqlite3_queries_made, 1, __ATOMIC_RELAXED);
if(success) {
__atomic_fetch_add(&global_statistics.sqlite3_queries_ok, 1, __ATOMIC_RELAXED);
}
else {
__atomic_fetch_add(&global_statistics.sqlite3_queries_failed, 1, __ATOMIC_RELAXED);
if(busy)
__atomic_fetch_add(&global_statistics.sqlite3_queries_failed_busy, 1, __ATOMIC_RELAXED);
if(locked)
__atomic_fetch_add(&global_statistics.sqlite3_queries_failed_locked, 1, __ATOMIC_RELAXED);
}
}
void sqlite3_row_completed(void) {
__atomic_fetch_add(&global_statistics.sqlite3_rows, 1, __ATOMIC_RELAXED);
}
void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_generated) {
__atomic_fetch_add(&global_statistics.rrdr_queries_made, 1, __ATOMIC_RELAXED);
__atomic_fetch_add(&global_statistics.rrdr_db_points_read, db_points_read, __ATOMIC_RELAXED);
@ -79,24 +108,31 @@ void web_client_disconnected(void) {
static inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) {
gs->connected_clients = __atomic_fetch_add(&global_statistics.connected_clients, 0, __ATOMIC_RELAXED);
gs->web_requests = __atomic_fetch_add(&global_statistics.web_requests, 0, __ATOMIC_RELAXED);
gs->web_usec = __atomic_fetch_add(&global_statistics.web_usec, 0, __ATOMIC_RELAXED);
gs->web_usec_max = __atomic_fetch_add(&global_statistics.web_usec_max, 0, __ATOMIC_RELAXED);
gs->bytes_received = __atomic_fetch_add(&global_statistics.bytes_received, 0, __ATOMIC_RELAXED);
gs->bytes_sent = __atomic_fetch_add(&global_statistics.bytes_sent, 0, __ATOMIC_RELAXED);
gs->content_size = __atomic_fetch_add(&global_statistics.content_size, 0, __ATOMIC_RELAXED);
gs->compressed_content_size = __atomic_fetch_add(&global_statistics.compressed_content_size, 0, __ATOMIC_RELAXED);
gs->web_client_count = __atomic_fetch_add(&global_statistics.web_client_count, 0, __ATOMIC_RELAXED);
gs->connected_clients = __atomic_load_n(&global_statistics.connected_clients, __ATOMIC_RELAXED);
gs->web_requests = __atomic_load_n(&global_statistics.web_requests, __ATOMIC_RELAXED);
gs->web_usec = __atomic_load_n(&global_statistics.web_usec, __ATOMIC_RELAXED);
gs->web_usec_max = __atomic_load_n(&global_statistics.web_usec_max, __ATOMIC_RELAXED);
gs->bytes_received = __atomic_load_n(&global_statistics.bytes_received, __ATOMIC_RELAXED);
gs->bytes_sent = __atomic_load_n(&global_statistics.bytes_sent, __ATOMIC_RELAXED);
gs->content_size = __atomic_load_n(&global_statistics.content_size, __ATOMIC_RELAXED);
gs->compressed_content_size = __atomic_load_n(&global_statistics.compressed_content_size, __ATOMIC_RELAXED);
gs->web_client_count = __atomic_load_n(&global_statistics.web_client_count, __ATOMIC_RELAXED);
gs->rrdr_queries_made = __atomic_fetch_add(&global_statistics.rrdr_queries_made, 0, __ATOMIC_RELAXED);
gs->rrdr_db_points_read = __atomic_fetch_add(&global_statistics.rrdr_db_points_read, 0, __ATOMIC_RELAXED);
gs->rrdr_result_points_generated = __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, 0, __ATOMIC_RELAXED);
gs->rrdr_queries_made = __atomic_load_n(&global_statistics.rrdr_queries_made, __ATOMIC_RELAXED);
gs->rrdr_db_points_read = __atomic_load_n(&global_statistics.rrdr_db_points_read, __ATOMIC_RELAXED);
gs->rrdr_result_points_generated = __atomic_load_n(&global_statistics.rrdr_result_points_generated, __ATOMIC_RELAXED);
if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX) {
uint64_t n = 0;
__atomic_compare_exchange(&global_statistics.web_usec_max, (uint64_t *) &gs->web_usec_max, &n, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
}
gs->sqlite3_queries_made = __atomic_load_n(&global_statistics.sqlite3_queries_made, __ATOMIC_RELAXED);
gs->sqlite3_queries_ok = __atomic_load_n(&global_statistics.sqlite3_queries_ok, __ATOMIC_RELAXED);
gs->sqlite3_queries_failed = __atomic_load_n(&global_statistics.sqlite3_queries_failed, __ATOMIC_RELAXED);
gs->sqlite3_queries_failed_busy = __atomic_load_n(&global_statistics.sqlite3_queries_failed_busy, __ATOMIC_RELAXED);
gs->sqlite3_queries_failed_locked = __atomic_load_n(&global_statistics.sqlite3_queries_failed_locked, __ATOMIC_RELAXED);
gs->sqlite3_rows = __atomic_load_n(&global_statistics.sqlite3_rows, __ATOMIC_RELAXED);
}
static void global_statistics_charts(void) {
@ -443,6 +479,106 @@ static void global_statistics_charts(void) {
}
// ----------------------------------------------------------------
if(gs.sqlite3_queries_made) {
static RRDSET *st_sqlite3_queries = NULL;
static RRDDIM *rd_queries = NULL;
if (unlikely(!st_sqlite3_queries)) {
st_sqlite3_queries = rrdset_create_localhost(
"netdata"
, "sqlite3_queries"
, NULL
, "sqlite3"
, NULL
, "Netdata SQLite3 Queries"
, "queries/s"
, "netdata"
, "stats"
, 131100
, localhost->rrd_update_every
, RRDSET_TYPE_LINE
);
rd_queries = rrddim_add(st_sqlite3_queries, "queries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
}
else
rrdset_next(st_sqlite3_queries);
rrddim_set_by_pointer(st_sqlite3_queries, rd_queries, (collected_number)gs.sqlite3_queries_made);
rrdset_done(st_sqlite3_queries);
}
// ----------------------------------------------------------------
if(gs.sqlite3_queries_ok || gs.sqlite3_queries_failed) {
static RRDSET *st_sqlite3_queries_by_status = NULL;
static RRDDIM *rd_ok = NULL, *rd_failed = NULL, *rd_busy = NULL, *rd_locked = NULL;
if (unlikely(!st_sqlite3_queries_by_status)) {
st_sqlite3_queries_by_status = rrdset_create_localhost(
"netdata"
, "sqlite3_queries_by_status"
, NULL
, "sqlite3"
, NULL
, "Netdata SQLite3 Queries by status"
, "queries/s"
, "netdata"
, "stats"
, 131101
, localhost->rrd_update_every
, RRDSET_TYPE_LINE
);
rd_ok = rrddim_add(st_sqlite3_queries_by_status, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_failed = rrddim_add(st_sqlite3_queries_by_status, "failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_busy = rrddim_add(st_sqlite3_queries_by_status, "busy", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
rd_locked = rrddim_add(st_sqlite3_queries_by_status, "locked", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
}
else
rrdset_next(st_sqlite3_queries_by_status);
rrddim_set_by_pointer(st_sqlite3_queries_by_status, rd_ok, (collected_number)gs.sqlite3_queries_made);
rrddim_set_by_pointer(st_sqlite3_queries_by_status, rd_failed, (collected_number)gs.sqlite3_queries_failed);
rrddim_set_by_pointer(st_sqlite3_queries_by_status, rd_busy, (collected_number)gs.sqlite3_queries_failed_busy);
rrddim_set_by_pointer(st_sqlite3_queries_by_status, rd_locked, (collected_number)gs.sqlite3_queries_failed_locked);
rrdset_done(st_sqlite3_queries_by_status);
}
// ----------------------------------------------------------------
if(gs.sqlite3_rows) {
static RRDSET *st_sqlite3_rows = NULL;
static RRDDIM *rd_rows = NULL;
if (unlikely(!st_sqlite3_rows)) {
st_sqlite3_rows = rrdset_create_localhost(
"netdata"
, "sqlite3_rows"
, NULL
, "sqlite3"
, NULL
, "Netdata SQLite3 Rows"
, "rows/s"
, "netdata"
, "stats"
, 131102
, localhost->rrd_update_every
, RRDSET_TYPE_LINE
);
rd_rows = rrddim_add(st_sqlite3_rows, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
}
else
rrdset_next(st_sqlite3_rows);
rrddim_set_by_pointer(st_sqlite3_rows, rd_rows, (collected_number)gs.sqlite3_rows);
rrdset_done(st_sqlite3_rows);
}
}
static void dbengine_statistics_charts(void) {

View File

@ -9,6 +9,8 @@
// global statistics
extern void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_generated);
extern void sqlite3_query_completed(bool success, bool busy, bool locked);
extern void sqlite3_row_completed(void);
extern void finished_web_request_statistics(uint64_t dt,
uint64_t bytes_received,

View File

@ -1527,19 +1527,19 @@ int test_sqlite(void) {
return 1;
}
rc = sqlite3_exec(db_meta, "CREATE TABLE IF NOT EXISTS mine (id1, id2);", 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, "CREATE TABLE IF NOT EXISTS mine (id1, id2);", 0, 0, NULL);
if (rc != SQLITE_OK) {
fprintf(stderr,"Failed to test SQLite: Create table failed\n");
return 1;
}
rc = sqlite3_exec(db_meta, "DELETE FROM MINE LIMIT 1;", 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, "DELETE FROM MINE LIMIT 1;", 0, 0, NULL);
if (rc != SQLITE_OK) {
fprintf(stderr,"Failed to test SQLite: Delete with LIMIT failed\n");
return 1;
}
rc = sqlite3_exec(db_meta, "UPDATE MINE SET id1=1 LIMIT 1;", 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, "UPDATE MINE SET id1=1 LIMIT 1;", 0, 0, NULL);
if (rc != SQLITE_OK) {
fprintf(stderr,"Failed to test SQLite: Update with LIMIT failed\n");
return 1;
@ -1549,49 +1549,49 @@ int test_sqlite(void) {
char *uuid_str = "0000_000";
buffer_sprintf(sql, TABLE_ACLK_CHART, uuid_str);
rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
buffer_flush(sql);
if (rc != SQLITE_OK)
goto error;
buffer_sprintf(sql, TABLE_ACLK_CHART_PAYLOAD, uuid_str);
rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
buffer_flush(sql);
if (rc != SQLITE_OK)
goto error;
buffer_sprintf(sql, TABLE_ACLK_CHART_LATEST, uuid_str);
rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
if (rc != SQLITE_OK)
goto error;
buffer_flush(sql);
buffer_sprintf(sql, INDEX_ACLK_CHART, uuid_str, uuid_str);
rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
if (rc != SQLITE_OK)
goto error;
buffer_flush(sql);
buffer_sprintf(sql, INDEX_ACLK_CHART_LATEST, uuid_str, uuid_str);
rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
if (rc != SQLITE_OK)
goto error;
buffer_flush(sql);
buffer_sprintf(sql, TRIGGER_ACLK_CHART_PAYLOAD, uuid_str, uuid_str, uuid_str);
rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
if (rc != SQLITE_OK)
goto error;
buffer_flush(sql);
buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str);
rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
if (rc != SQLITE_OK)
goto error;
buffer_flush(sql);
buffer_sprintf(sql, INDEX_ACLK_ALERT, uuid_str, uuid_str);
rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
if (rc != SQLITE_OK)
goto error;
buffer_flush(sql);

View File

@ -361,7 +361,7 @@ void sql_aclk_sync_init(void)
for (int i = 0; aclk_sync_config[i]; i++) {
debug(D_ACLK_SYNC, "Executing %s", aclk_sync_config[i]);
rc = sqlite3_exec(db_meta, aclk_sync_config[i], 0, 0, &err_msg);
rc = sqlite3_exec_monitored(db_meta, aclk_sync_config[i], 0, 0, &err_msg);
if (rc != SQLITE_OK) {
error_report("SQLite error aclk sync initialization setup, rc = %d (%s)", rc, err_msg);
error_report("SQLite failed statement %s", aclk_sync_config[i]);
@ -373,7 +373,7 @@ void sql_aclk_sync_init(void)
fatal_assert(0 == uv_mutex_init(&aclk_async_lock));
if (likely(rrdcontext_enabled == CONFIG_BOOLEAN_YES)) {
rc = sqlite3_exec(db_meta, "SELECT host_id, hostname, registry_hostname, update_every, os, "
rc = sqlite3_exec_monitored(db_meta, "SELECT host_id, hostname, registry_hostname, update_every, os, "
"timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, "
"program_version, entries, health_enabled FROM host WHERE hops >0;",
create_host_callback, NULL, &err_msg);
@ -383,7 +383,7 @@ void sql_aclk_sync_init(void)
}
}
rc = sqlite3_exec(db_meta, "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni WHERE "
rc = sqlite3_exec_monitored(db_meta, "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni WHERE "
"h.host_id = ni.host_id AND ni.node_id IS NOT NULL;", aclk_start_sync_thread, NULL, &err_msg);
if (rc != SQLITE_OK) {
error_report("SQLite error when starting ACLK sync threads, rc = %d (%s)", rc, err_msg);
@ -927,7 +927,7 @@ static int is_host_available(uuid_t *host_id)
error_report("Failed to bind host_id parameter to select node instance information");
goto failed;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
failed:
if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
@ -980,7 +980,7 @@ void sql_delete_aclk_table_list(struct aclk_database_worker_config *wc, struct a
}
buffer_flush(sql);
while (sqlite3_step(res) == SQLITE_ROW)
while (sqlite3_step_monitored(res) == SQLITE_ROW)
buffer_strcat(sql, (char *) sqlite3_column_text(res, 0));
rc = sqlite3_finalize(res);
@ -1016,7 +1016,7 @@ void sql_check_aclk_table_list(struct aclk_database_worker_config *wc)
{
char *err_msg = NULL;
debug(D_ACLK_SYNC,"Cleaning tables for nodes that do not exist");
int rc = sqlite3_exec(db_meta, SQL_SELECT_ACLK_ACTIVE_LIST, sql_check_aclk_table, (void *) wc, &err_msg);
int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ACTIVE_LIST, sql_check_aclk_table, (void *) wc, &err_msg);
if (rc != SQLITE_OK) {
error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
sqlite3_free(err_msg);

View File

@ -24,7 +24,7 @@ time_t removed_when(uint32_t alarm_id, uint32_t before_unique_id, uint32_t after
return 0;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW)) {
when = (time_t) sqlite3_column_int64(res, 0);
}
@ -70,7 +70,7 @@ int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)
return send;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW)) {
status = (RRDCALC_STATUS) sqlite3_column_int(res, 0);
if (sqlite3_column_type(res, 1) != SQLITE_NULL)
@ -280,7 +280,7 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
static __thread uint64_t log_first_sequence_id = 0;
static __thread uint64_t log_last_sequence_id = 0;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
struct alarm_log_entry alarm_log;
char old_value_string[100 + 1];
char new_value_string[100 + 1];
@ -500,7 +500,7 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
last_timestamp.tv_sec = 0;
last_timestamp.tv_usec = 0;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
first_sequence = sqlite3_column_bytes(res, 0) > 0 ? (uint64_t) sqlite3_column_int64(res, 0) : 0;
if (sqlite3_column_bytes(res, 1) > 0) {
first_timestamp.tv_sec = sqlite3_column_int64(res, 1);
@ -603,7 +603,7 @@ int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct
struct provide_alarm_configuration p_alarm_config;
p_alarm_config.cfg_hash = NULL;
if (sqlite3_step(res) == SQLITE_ROW) {
if (sqlite3_step_monitored(res) == SQLITE_ROW) {
alarm_config.alarm = sqlite3_column_bytes(res, 0) > 0 ? strdupz((char *)sqlite3_column_text(res, 0)) : NULL;
alarm_config.tmpl = sqlite3_column_bytes(res, 1) > 0 ? strdupz((char *)sqlite3_column_text(res, 1)) : NULL;
@ -1029,7 +1029,7 @@ void sql_aclk_alert_clean_dead_entries(RRDHOST *host)
" (select unique_id from health_log_%s); ", uuid_str, uuid_str);
char *err_msg = NULL;
int rc = sqlite3_exec(db_meta, buffer_tostring(sql), NULL, NULL, &err_msg);
int rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), NULL, NULL, &err_msg);
if (rc != SQLITE_OK) {
error_report("Failed when trying to clean stale ACLK alert entries from aclk_alert_%s, error message \"%s""",
uuid_str, err_msg);
@ -1064,7 +1064,7 @@ int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert
return 1;
}
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
proto_alert_status->pending_min_sequence_id = sqlite3_column_bytes(res, 0) > 0 ? (uint64_t) sqlite3_column_int64(res, 0) : 0;
proto_alert_status->pending_max_sequence_id = sqlite3_column_bytes(res, 1) > 0 ? (uint64_t) sqlite3_column_int64(res, 1) : 0;
proto_alert_status->last_acked_sequence_id = sqlite3_column_bytes(res, 2) > 0 ? (uint64_t) sqlite3_column_int64(res, 2) : 0;

View File

@ -48,7 +48,7 @@ static time_t payload_sent(char *uuid_str, uuid_t *uuid, void *payload, size_t p
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
send_status = (time_t) sqlite3_column_int64(res, 0);
}
@ -245,7 +245,7 @@ void aclk_process_dimension_deletion(struct aclk_database_worker_config *wc, str
goto bind_fail;
unsigned count = 0;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
(void) aclk_upd_dimension_event(
wc,
claim_id,
@ -357,7 +357,7 @@ void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_d
int count = 0;
first_sequence = 0;
last_sequence = 0;
while (count < limit && sqlite3_step(res) == SQLITE_ROW) {
while (count < limit && sqlite3_step_monitored(res) == SQLITE_ROW) {
size_t payload_size = sqlite3_column_bytes(res, 1);
if (payload_list_max_size[count] < payload_size) {
freez(payload_list[count]);
@ -487,7 +487,7 @@ int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_d
struct chart_config_updated chart_config;
chart_config.config_hash = NULL;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
chart_config.type = strdupz((char *)sqlite3_column_text(res, 0));
chart_config.family = strdupz((char *)sqlite3_column_text(res, 1));
chart_config.context = strdupz((char *)sqlite3_column_text(res, 2));
@ -799,7 +799,7 @@ static RRD_MEMORY_MODE sql_get_host_memory_mode(uuid_t *host_id)
goto failed;
}
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
memory_mode = (RRD_MEMORY_MODE)sqlite3_column_int(res, 0);
}
@ -891,7 +891,7 @@ void aclk_update_retention(struct aclk_database_worker_config *wc)
rotate_data.node_id = strdupz(wc->node_id);
time_t now = now_realtime_sec();
while (sqlite3_step(res) == SQLITE_ROW && dimension_update_count < ACLK_MAX_DIMENSION_CLEANUP) {
while (sqlite3_step_monitored(res) == SQLITE_ROW && dimension_update_count < ACLK_MAX_DIMENSION_CLEANUP) {
if (unlikely(netdata_exit))
break;
if (!update_every || update_every != (uint32_t)sqlite3_column_int(res, 1)) {
@ -1022,7 +1022,7 @@ uint32_t sql_get_pending_count(struct aclk_database_worker_config *wc)
return 0;
}
}
while (sqlite3_step(res) == SQLITE_ROW)
while (sqlite3_step_monitored(res) == SQLITE_ROW)
chart_payload_count = (uint32_t) sqlite3_column_int(res, 0);
rc = sqlite3_reset(res);
@ -1049,7 +1049,7 @@ void sql_get_last_chart_sequence(struct aclk_database_worker_config *wc)
wc->chart_sequence_id = 0;
wc->chart_timestamp = 0;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
wc->chart_sequence_id = (uint64_t)sqlite3_column_int64(res, 0);
wc->chart_timestamp = (time_t)sqlite3_column_int64(res, 1);
}
@ -1220,37 +1220,37 @@ struct aclk_chart_sync_stats *aclk_get_chart_sync_stats(RRDHOST *host)
return NULL;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (rc == SQLITE_ROW) {
aclk_statistics->min_seqid = SQL_SEQ_NULL(res, 0);
aclk_statistics->max_seqid = SQL_SEQ_NULL(res, 1);
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (rc == SQLITE_ROW) {
aclk_statistics->min_seqid_pend = SQL_SEQ_NULL(res, 0);
aclk_statistics->max_seqid_pend = SQL_SEQ_NULL(res, 1);
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (rc == SQLITE_ROW) {
aclk_statistics->min_seqid_sent = SQL_SEQ_NULL(res, 0);
aclk_statistics->max_seqid_sent = SQL_SEQ_NULL(res, 1);
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (rc == SQLITE_ROW) {
aclk_statistics->min_seqid_ack = SQL_SEQ_NULL(res, 0);
aclk_statistics->max_seqid_ack = SQL_SEQ_NULL(res, 1);
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (rc == SQLITE_ROW) {
aclk_statistics->min_seqid_ack = SQL_SEQ_NULL(res, 0);
aclk_statistics->max_seqid_ack = SQL_SEQ_NULL(res, 1);
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (rc == SQLITE_ROW) {
aclk_statistics->max_date_created = (time_t) SQL_SEQ_NULL(res, 0);
aclk_statistics->max_date_submitted = (time_t) SQL_SEQ_NULL(res, 1);

View File

@ -150,7 +150,7 @@ void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, voi
}
SQL_CHART_DATA chart_data = { 0 };
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
uuid_copy(chart_data.chart_id, *((uuid_t *)sqlite3_column_blob(res, 0)));
chart_data.id = (char *) sqlite3_column_text(res, 1);
chart_data.name = (char *) sqlite3_column_text(res, 2);
@ -191,7 +191,7 @@ void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DA
SQL_DIMENSION_DATA dimension_data;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
uuid_copy(dimension_data.dim_id, *((uuid_t *)sqlite3_column_blob(res, 0)));
dimension_data.id = (char *) sqlite3_column_text(res, 1);
dimension_data.name = (char *) sqlite3_column_text(res, 2);
@ -225,7 +225,7 @@ void ctx_get_label_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, v
SQL_CLABEL_DATA label_data;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
label_data.label_key = (char *) sqlite3_column_text(res, 0);
label_data.label_value = (char *) sqlite3_column_text(res, 1);
label_data.label_source = sqlite3_column_int(res, 2);
@ -267,7 +267,7 @@ void ctx_get_context_list(uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEXT_D
goto failed;
}
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
context_data.id = (char *) sqlite3_column_text(res, 0);
context_data.version = sqlite3_column_int64(res, 1);
context_data.title = (char *) sqlite3_column_text(res, 2);

View File

@ -21,7 +21,7 @@ static int table_exists_in_database(const char *table)
snprintf(sql, 127, "select 1 from sqlite_schema where type = 'table' and name = '%s';", table);
int rc = sqlite3_exec(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
int rc = sqlite3_exec_monitored(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
if (rc != SQLITE_OK) {
info("Error checking table existence; %s", err_msg);
sqlite3_free(err_msg);
@ -39,7 +39,7 @@ static int column_exists_in_table(const char *table, const char *column)
snprintf(sql, 127, "SELECT 1 FROM pragma_table_info('%s') where name = '%s';", table, column);
int rc = sqlite3_exec(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
int rc = sqlite3_exec_monitored(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
if (rc != SQLITE_OK) {
info("Error checking column existence; %s", err_msg);
sqlite3_free(err_msg);
@ -100,11 +100,11 @@ static int do_migration_v3_v4(sqlite3 *database, const char *name)
return 1;
}
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
char *table = strdupz((char *) sqlite3_column_text(res, 0));
if (!column_exists_in_table(table, "chart_context")) {
snprintfz(sql, 255, "ALTER TABLE %s ADD chart_context text", table);
sqlite3_exec(database, sql, 0, 0, NULL);
sqlite3_exec_monitored(database, sql, 0, 0, NULL);
}
freez(table);
}
@ -135,7 +135,7 @@ static int migrate_database(sqlite3 *database, int target_version, char *db_name
int user_version = 0;
char *err_msg = NULL;
int rc = sqlite3_exec(database, "PRAGMA user_version;", return_int_cb, (void *) &user_version, &err_msg);
int rc = sqlite3_exec_monitored(database, "PRAGMA user_version;", return_int_cb, (void *) &user_version, &err_msg);
if (rc != SQLITE_OK) {
info("Error checking the %s database version; %s", db_name, err_msg);
sqlite3_free(err_msg);

View File

@ -88,11 +88,35 @@ pthread_key_t key_pool[MAX_PREPARED_STATEMENTS];
static uv_mutex_t sqlite_transaction_lock;
SQLITE_API int sqlite3_exec_monitored(
sqlite3 *db, /* An open database */
const char *sql, /* SQL to be evaluated */
int (*callback)(void*,int,char**,char**), /* Callback function */
void *data, /* 1st argument to callback */
char **errmsg /* Error msg written here */
) {
int rc = sqlite3_exec(db, sql, callback, data, errmsg);
sqlite3_query_completed(rc == SQLITE_OK, rc == SQLITE_BUSY, rc == SQLITE_LOCKED);
return rc;
}
SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt) {
int rc = sqlite3_step(stmt);
if(likely(rc == SQLITE_ROW))
sqlite3_row_completed();
else
sqlite3_query_completed(rc == SQLITE_DONE, rc == SQLITE_BUSY, rc == SQLITE_LOCKED);
return rc;
}
int execute_insert(sqlite3_stmt *res)
{
int rc;
int cnt = 0;
while ((rc = sqlite3_step(res)) != SQLITE_DONE && ++cnt < SQL_MAX_RETRY && likely(!netdata_exit)) {
while ((rc = sqlite3_step_monitored(res)) != SQLITE_DONE && ++cnt < SQL_MAX_RETRY && likely(!netdata_exit)) {
if (likely(rc == SQLITE_BUSY || rc == SQLITE_LOCKED)) {
usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
error_report("Failed to insert/update, rc = %d -- attempt %d", rc, cnt);
@ -273,7 +297,7 @@ static int check_table_integrity(char *table)
strcpy(wstr,"PRAGMA integrity_check;");
}
int rc = sqlite3_exec(db_meta, wstr, check_table_integrity_cb, (void *) &status, &err_msg);
int rc = sqlite3_exec_monitored(db_meta, wstr, check_table_integrity_cb, (void *) &status, &err_msg);
if (rc != SQLITE_OK) {
error_report("SQLite error during database integrity check for %s, rc = %d (%s)",
table ? table : "the entire database", rc, err_msg);
@ -306,7 +330,7 @@ static void rebuild_chart()
info("Rebuilding chart table");
for (int i = 0; rebuild_chart_commands[i]; i++) {
info("Executing %s", rebuild_chart_commands[i]);
rc = sqlite3_exec(db_meta, rebuild_chart_commands[i], 0, 0, &err_msg);
rc = sqlite3_exec_monitored(db_meta, rebuild_chart_commands[i], 0, 0, &err_msg);
if (rc != SQLITE_OK) {
error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);
error_report("SQLite failed statement %s", rebuild_chart_commands[i]);
@ -339,7 +363,7 @@ void rebuild_dimension()
info("Rebuilding dimension table");
for (int i = 0; rebuild_dimension_commands[i]; i++) {
info("Executing %s", rebuild_dimension_commands[i]);
rc = sqlite3_exec(db_meta, rebuild_dimension_commands[i], 0, 0, &err_msg);
rc = sqlite3_exec_monitored(db_meta, rebuild_dimension_commands[i], 0, 0, &err_msg);
if (rc != SQLITE_OK) {
error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);
error_report("SQLite failed statement %s", rebuild_dimension_commands[i]);
@ -366,7 +390,7 @@ int init_database_batch(sqlite3 *database, int rebuild, int init_type, const cha
char *err_msg = NULL;
for (int i = 0; batch[i]; i++) {
debug(D_METADATALOG, "Executing %s", batch[i]);
rc = sqlite3_exec(database, batch[i], 0, 0, &err_msg);
rc = sqlite3_exec_monitored(database, batch[i], 0, 0, &err_msg);
if (rc != SQLITE_OK) {
error_report("SQLite error during database %s, rc = %d (%s)", init_type ? "cleanup" : "setup", rc, err_msg);
error_report("SQLite failed statement %s", batch[i]);
@ -437,7 +461,7 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
if (rebuild & DB_CHECK_RECLAIM_SPACE) {
if (!(rebuild & DB_CHECK_CONT))
info("Reclaiming space of %s", sqlite_database);
rc = sqlite3_exec(db_meta, "VACUUM;", 0, 0, &err_msg);
rc = sqlite3_exec_monitored(db_meta, "VACUUM;", 0, 0, &err_msg);
if (rc != SQLITE_OK) {
error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);
sqlite3_free(err_msg);
@ -546,7 +570,7 @@ int find_uuid_type(uuid_t *uuid)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW))
uuid_type = sqlite3_column_int(res, 0);
@ -589,7 +613,7 @@ int find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW)) {
uuid_copy(*store_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
status = 0;
@ -636,7 +660,7 @@ void delete_dimension_uuid(uuid_t *dimension_uuid)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (unlikely(rc != SQLITE_DONE))
error_report("Failed to delete dimension uuid, rc = %d", rc);
@ -684,7 +708,7 @@ uuid_t *find_chart_uuid(RRDHOST *host, const char *type, const char *id, const c
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW)) {
uuid = mallocz(sizeof(uuid_t));
uuid_copy(*uuid, sqlite3_column_blob(res, 0));
@ -847,7 +871,7 @@ int sql_store_host(
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
int store_rc = sqlite3_step(res);
int store_rc = sqlite3_step_monitored(res);
if (unlikely(store_rc != SQLITE_DONE))
error_report("Failed to store host %s, rc = %d", hostname, rc);
@ -954,7 +978,7 @@ int sql_store_host_info(RRDHOST *host)
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
int store_rc = sqlite3_step(res);
int store_rc = sqlite3_step_monitored(res);
if (unlikely(store_rc != SQLITE_DONE))
error_report("Failed to store host %s, rc = %d", host->hostname, rc);
@ -1229,7 +1253,7 @@ void sql_rrdim2json(sqlite3_stmt *res_dim, uuid_t *chart_uuid, BUFFER *wb, size_
int dimensions = 0;
buffer_sprintf(wb, "\t\t\t\"dimensions\": {\n");
while (sqlite3_step(res_dim) == SQLITE_ROW) {
while (sqlite3_step_monitored(res_dim) == SQLITE_ROW) {
if (dimensions)
buffer_strcat(wb, ",\n\t\t\t\t\"");
else
@ -1305,7 +1329,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb)
size_t c = 0;
size_t dimensions = 0;
while (sqlite3_step(res_chart) == SQLITE_ROW) {
while (sqlite3_step_monitored(res_chart) == SQLITE_ROW) {
char id[512];
sprintf(id, "%s.%s", sqlite3_column_text(res_chart, 3), sqlite3_column_text(res_chart, 1));
RRDSET *st = rrdset_find(host, id);
@ -1471,7 +1495,7 @@ RRDHOST *sql_create_host_by_uuid(char *hostname)
}
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (unlikely(rc != SQLITE_ROW)) {
error_report("Failed to find hostname %s", hostname);
goto failed;
@ -1511,20 +1535,21 @@ void db_execute(const char *cmd)
int cnt = 0;
while (cnt < SQL_MAX_RETRY) {
char *err_msg;
rc = sqlite3_exec(db_meta, cmd, 0, 0, &err_msg);
rc = sqlite3_exec_monitored(db_meta, cmd, 0, 0, &err_msg);
if (rc != SQLITE_OK) {
error_report("Failed to execute '%s', rc = %d (%s) -- attempt %d", cmd, rc, err_msg, cnt);
sqlite3_free(err_msg);
if (likely(rc == SQLITE_BUSY || rc == SQLITE_LOCKED)) {
usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
}
else break;
else
break;
}
else
break;
++cnt;
}
return;
}
void db_lock(void)
@ -1559,7 +1584,7 @@ int file_is_migrated(char *path)
return 0;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
error_report("Failed to finalize the prepared statement when checking if metadata file is migrated");
@ -1838,7 +1863,7 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para
uuid_t rrdeng_uuid;
uuid_t chart_id;
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
char id[512];
sprintf(id, "%s.%s", sqlite3_column_text(res, 3), sqlite3_column_text(res, 1));
@ -2242,7 +2267,7 @@ char *get_hostname_by_node_id(char *node)
goto failed;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW))
hostname = strdupz((char *)sqlite3_column_text(res, 0));
@ -2280,7 +2305,7 @@ int get_host_id(uuid_t *node_id, uuid_t *host_id)
goto failed;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW && host_id))
uuid_copy(*host_id, *((uuid_t *) sqlite3_column_blob(res, 0)));
@ -2316,7 +2341,7 @@ int get_node_id(uuid_t *host_id, uuid_t *node_id)
goto failed;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW && node_id))
uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0)));
@ -2395,7 +2420,7 @@ struct node_instance_list *get_node_list(void)
int row = 0;
char host_guid[37];
while (sqlite3_step(res) == SQLITE_ROW)
while (sqlite3_step_monitored(res) == SQLITE_ROW)
row++;
if (sqlite3_reset(res) != SQLITE_OK) {
@ -2406,7 +2431,7 @@ struct node_instance_list *get_node_list(void)
int max_rows = row;
row = 0;
rrd_rdlock();
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t))
uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0)));
if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) {
@ -2461,7 +2486,7 @@ void sql_load_node_id(RRDHOST *host)
goto failed;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW)) {
if (likely(sqlite3_column_bytes(res, 0) == sizeof(uuid_t)))
set_host_node_id(host, (uuid_t *)sqlite3_column_blob(res, 0));
@ -2497,7 +2522,7 @@ void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *sys
goto skip_loading;
}
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
rrdhost_set_system_info_variable(system_info, (char *) sqlite3_column_text(res, 0),
(char *) sqlite3_column_text(res, 1));
}
@ -2681,7 +2706,7 @@ DICTIONARY *sql_load_host_labels(uuid_t *host_id)
labels = rrdlabels_create();
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
rrdlabels_add(
labels,
(const char *)sqlite3_column_text(res, 0),

View File

@ -58,6 +58,15 @@ typedef enum db_check_action_type {
return 1; \
}
extern SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt);
extern SQLITE_API int sqlite3_exec_monitored(
sqlite3 *db, /* An open database */
const char *sql, /* SQL to be evaluated */
int (*callback)(void*,int,char**,char**), /* Callback function */
void *data, /* 1st argument to callback */
char **errmsg /* Error msg written here */
);
extern int sql_init_database(db_check_action_type_t rebuild, int memory);
extern void sql_close_database(void);
extern int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null);

View File

@ -24,7 +24,7 @@ int sql_create_health_log_table(RRDHOST *host) {
snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CREATE_HEALTH_LOG_TABLE(uuid_str));
rc = sqlite3_exec(db_meta, command, 0, 0, &err_msg);
rc = sqlite3_exec_monitored(db_meta, command, 0, 0, &err_msg);
if (rc != SQLITE_OK) {
error_report("HEALTH [%s]: SQLite error during creation of health log table, rc = %d (%s)", host->hostname, rc, err_msg);
sqlite3_free(err_msg);
@ -389,7 +389,7 @@ void sql_health_alarm_log_cleanup(RRDHOST *host) {
return;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (unlikely(rc != SQLITE_DONE))
error_report("Failed to cleanup health log table, rc = %d", rc);
@ -428,7 +428,7 @@ void sql_health_alarm_log_count(RRDHOST *host) {
return;
}
rc = sqlite3_step(res);
rc = sqlite3_step_monitored(res);
if (likely(rc == SQLITE_ROW))
host->health_log_entries_written = (size_t) sqlite3_column_int64(res, 0);
@ -556,7 +556,7 @@ uint32_t sql_get_max_unique_id (char *uuid_str)
return 0;
}
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
max_unique_id = (uint32_t) sqlite3_column_int64(res, 0);
}
@ -584,7 +584,7 @@ void sql_check_removed_alerts_state(char *uuid_str)
return;
}
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
status = (RRDCALC_STATUS) sqlite3_column_int(res, 0);
unique_id = (uint32_t) sqlite3_column_int64(res, 1);
alarm_id = (uint32_t) sqlite3_column_int64(res, 2);
@ -634,7 +634,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
while (sqlite3_step(res) == SQLITE_ROW) {
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
ALARM_ENTRY *ae = NULL;
// check that we have valid ids

View File

@ -105,7 +105,7 @@ Database::Database(const std::string &Path) {
// Create anomaly events table if it does not exist.
char *ErrMsg;
RC = sqlite3_exec(Conn, SQL_CREATE_ANOMALIES_TABLE, nullptr, nullptr, &ErrMsg);
RC = sqlite3_exec_monitored(Conn, SQL_CREATE_ANOMALIES_TABLE, nullptr, nullptr, &ErrMsg);
if (RC == SQLITE_OK)
return;

View File

@ -32,7 +32,7 @@ public:
}
while (true) {
switch (int RC = sqlite3_step(ParsedStmt)) {
switch (int RC = sqlite3_step_monitored(ParsedStmt)) {
case SQLITE_BUSY: case SQLITE_LOCKED:
usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
continue;