From 0b7ba99b5e344620ea3f1e633ba574590b9eda82 Mon Sep 17 00:00:00 2001 From: Adrien Mahieux Date: Mon, 21 Oct 2019 09:51:46 +0200 Subject: [PATCH] [collector/proc.plugin] Add /proc/pagetypeinfo parser (#6843) * [proc.plugin/proc_pagetypeinfo] Initial commit * [Fix] Generate graphs for pagetypeinfo * [Fix] Create node/zone/type graphs * [Fix] Use directly size and order * [Add] Configuration handling * [Imp] Changed SetId to identify NodeNumber * [Fix] Standard name for chart priority and value * [Fix] use dynamic pagesize * [Enh] allow prefix for containerized netdata * [Fix] global system graph always on, but for explicit no * [Fix] Add more checks for pageorders_cnt and really use it * [Enh] Special config value of netdata_zero_metrics_enabled * [Fix] Check we parsed at least a valid line --- CMakeLists.txt | 1 + Makefile.am | 1 + collectors/all.h | 1 + collectors/proc.plugin/plugin_proc.c | 1 + collectors/proc.plugin/plugin_proc.h | 1 + collectors/proc.plugin/proc_pagetypeinfo.c | 333 +++++++++++++++++++++ 6 files changed, 338 insertions(+) create mode 100644 collectors/proc.plugin/proc_pagetypeinfo.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 857f5b9af9..5158cdeba2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -415,6 +415,7 @@ set(PROC_PLUGIN_FILES collectors/proc.plugin/proc_softirqs.c collectors/proc.plugin/proc_loadavg.c collectors/proc.plugin/proc_meminfo.c + collectors/proc.plugin/proc_pagetypeinfo.c collectors/proc.plugin/proc_net_dev.c collectors/proc.plugin/proc_net_ip_vs_stats.c collectors/proc.plugin/proc_net_netstat.c diff --git a/Makefile.am b/Makefile.am index d9da7b8a0b..ccd26d1f7f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -262,6 +262,7 @@ PROC_PLUGIN_FILES = \ collectors/proc.plugin/proc_softirqs.c \ collectors/proc.plugin/proc_loadavg.c \ collectors/proc.plugin/proc_meminfo.c \ + collectors/proc.plugin/proc_pagetypeinfo.c \ collectors/proc.plugin/proc_net_dev.c \ collectors/proc.plugin/proc_net_ip_vs_stats.c \ collectors/proc.plugin/proc_net_netstat.c \ diff --git a/collectors/all.h b/collectors/all.h index 5fe5e9be1b..f58f0dcd5e 100644 --- a/collectors/all.h +++ b/collectors/all.h @@ -90,6 +90,7 @@ #define NETDATA_CHART_PRIO_MEM_KSM_RATIOS 1302 #define NETDATA_CHART_PRIO_MEM_NUMA 1400 #define NETDATA_CHART_PRIO_MEM_NUMA_NODES 1410 +#define NETDATA_CHART_PRIO_MEM_PAGEFRAG 1450 #define NETDATA_CHART_PRIO_MEM_HW 1500 #define NETDATA_CHART_PRIO_MEM_HW_ECC_CE 1550 #define NETDATA_CHART_PRIO_MEM_HW_ECC_UE 1560 diff --git a/collectors/proc.plugin/plugin_proc.c b/collectors/proc.plugin/plugin_proc.c index 62e974cfd0..116717408c 100644 --- a/collectors/proc.plugin/plugin_proc.c +++ b/collectors/proc.plugin/plugin_proc.c @@ -32,6 +32,7 @@ static struct proc_module { { .name = "/sys/block/zram", .dim = "zram", .func = do_sys_block_zram }, { .name = "/sys/devices/system/edac/mc", .dim = "ecc", .func = do_proc_sys_devices_system_edac_mc }, { .name = "/sys/devices/system/node", .dim = "numa", .func = do_proc_sys_devices_system_node }, + { .name = "/proc/pagetypeinfo", .dim = "pagetypeinfo", .func = do_proc_pagetypeinfo }, // network metrics { .name = "/proc/net/dev", .dim = "netdev", .func = do_proc_net_dev }, diff --git a/collectors/proc.plugin/plugin_proc.h b/collectors/proc.plugin/plugin_proc.h index 40a0e82d39..cb9a0c5f34 100644 --- a/collectors/proc.plugin/plugin_proc.h +++ b/collectors/proc.plugin/plugin_proc.h @@ -55,6 +55,7 @@ extern int do_proc_net_sockstat6(int update_every, usec_t dt); extern int do_proc_net_sctp_snmp(int update_every, usec_t dt); extern int do_ipc(int update_every, usec_t dt); extern int do_sys_class_power_supply(int update_every, usec_t dt); +extern int do_proc_pagetypeinfo(int update_every, usec_t dt); extern int get_numa_node_count(void); // metrics that need to be shared among data collectors diff --git a/collectors/proc.plugin/proc_pagetypeinfo.c b/collectors/proc.plugin/proc_pagetypeinfo.c new file mode 100644 index 0000000000..2fdadffee0 --- /dev/null +++ b/collectors/proc.plugin/proc_pagetypeinfo.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "plugin_proc.h" + +// For ULONG_MAX +#include + +#define PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME "/proc/pagetypeinfo" +#define CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME + +// Zone struct is pglist_data, in include/linux/mmzone.h +// MAX_NR_ZONES is from __MAX_NR_ZONE, which is the last value of the enum. +#define MAX_PAGETYPE_ORDER 11 + +// Names are in mm/page_alloc.c :: migratetype_names. Max size = 10. +#define MAX_ZONETYPE_NAME 16 +#define MAX_PAGETYPE_NAME 16 + +// Defined in include/linux/mmzone.h as __MAX_NR_ZONE (last enum of zone_type) +#define MAX_ZONETYPE 6 +// Defined in include/linux/mmzone.h as MIGRATE_TYPES (last enum of migratetype) +#define MAX_PAGETYPE 7 + + +// +// /proc/pagetypeinfo is declared in mm/vmstat.c :: init_mm_internals +// + +// One line of /proc/pagetypeinfo +struct pageline { + int node; + char *zone; + char *type; + int line; + uint64_t free_pages_size[MAX_PAGETYPE_ORDER]; + RRDDIM *rd[MAX_PAGETYPE_ORDER]; +}; + +// Sum of all orders +struct systemorder { + uint64_t size; + RRDDIM *rd; +}; + + +static inline uint64_t pageline_total_count(struct pageline *p) { + uint64_t sum = 0, o; + for (o=0; ofree_pages_size[o]; + return sum; +} + +// Check if a line of /proc/pagetypeinfo is valid to use +// Free block lines starts by "Node" && 4th col is "type" +#define pagetypeinfo_line_valid(ff, l) (strncmp(procfile_lineword(ff, l, 0), "Node", 4) == 0 && strncmp(procfile_lineword(ff, l, 4), "type", 4) == 0) + +// Dimension name from the order +#define dim_name(s, o, pagesize) (snprintfz(s, 16,"%ldKB (%lu)", (1 << o) * pagesize / 1024, o)) + +int do_proc_pagetypeinfo(int update_every, usec_t dt) { + (void)dt; + + // Config + static int do_global, do_detail; + static SIMPLE_PATTERN *filter_types = NULL; + + // Counters from parsing the file, that doesn't change after boot + static struct systemorder systemorders[MAX_PAGETYPE_ORDER] = {}; + static struct pageline* pagelines = NULL; + static long pagesize = 0; + static size_t pageorders_cnt = 0, pagelines_cnt = 0, ff_lines = 0; + + // Handle + static procfile *ff = NULL; + static char ff_path[FILENAME_MAX + 1]; + + // RRD Sets + static RRDSET *st_order = NULL; + static RRDSET **st_nodezonetype = NULL; + + // Local temp variables + size_t l, o, p; + struct pageline *pgl = NULL; + + // -------------------------------------------------------------------- + // Startup: Init arch and open /proc/pagetypeinfo + if (unlikely(!pagesize)) { + pagesize = sysconf(_SC_PAGESIZE); + } + + if(unlikely(!ff)) { + snprintfz(ff_path, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME); + ff = procfile_open(config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "filename to monitor", ff_path), " \t:", PROCFILE_FLAG_DEFAULT); + + if(unlikely(!ff)) { + strncpyz(ff_path, PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME, FILENAME_MAX); + ff = procfile_open(PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME, " \t,", PROCFILE_FLAG_DEFAULT); + } + } + if(unlikely(!ff)) + return 1; + + ff = procfile_readall(ff); + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time + + // -------------------------------------------------------------------- + // Init: find how many Nodes, Zones and Types + if(unlikely(pagelines_cnt == 0)) { + size_t nodenumlast = -1; + char *zonenamelast = NULL; + + ff_lines = procfile_lines(ff); + if(unlikely(!ff_lines)) { + error("PLUGIN: PROC_PAGETYPEINFO: Cannot read %s, zero lines reported.", ff_path); + return 1; + } + + // Configuration + do_global = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable system summary", CONFIG_BOOLEAN_YES); + do_detail = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable detail per-type", CONFIG_BOOLEAN_AUTO); + filter_types = simple_pattern_create( + config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "hide charts id matching", "") + , NULL + , SIMPLE_PATTERN_SUFFIX + ); + + pagelines_cnt = 0; + + // Pass 1: how many lines would be valid + for (l = 4; l < ff_lines; l++) { + if (!pagetypeinfo_line_valid(ff, l)) + continue; + + pagelines_cnt++; + } + if (pagelines_cnt == 0) { + error("PLUGIN: PROC_PAGETYPEINFO: Unable to parse any valid line in %s", ff_path); + return 1; + } + + // 4th line is the "Free pages count per migrate type at order". Just substract these 8 words. + pageorders_cnt = procfile_linewords(ff, 3); + if (pageorders_cnt < 9) { + error("PLUGIN: PROC_PAGETYPEINFO: Unable to parse Line 4 of %s", ff_path); + return 1; + } + + pageorders_cnt -= 9; + + if (pageorders_cnt > MAX_PAGETYPE_ORDER) { + error("PLUGIN: PROC_PAGETYPEINFO: pageorder found (%lu) is higher than max %d", pageorders_cnt, MAX_PAGETYPE_ORDER); + return 1; + } + + // Init pagelines from scanned lines + if (!pagelines) { + pagelines = callocz(pagelines_cnt, sizeof(struct pageline)); + if (!pagelines) { + error("PLUGIN: PROC_PAGETYPEINFO: Cannot allocate %lu pagelines of %lu B", pagelines_cnt, sizeof(struct pageline)); + return 1; + } + } + + // Pass 2: Scan the file again, with details + p = 0; + for (l=4; l < ff_lines; l++) { + + if (!pagetypeinfo_line_valid(ff, l)) + continue; + + size_t nodenum = strtoul(procfile_lineword(ff, l, 1), NULL, 10); + char *zonename = procfile_lineword(ff, l, 3); + char *typename = procfile_lineword(ff, l, 5); + + // We changed node or zone + if (nodenum != nodenumlast || !zonenamelast || strncmp(zonename, zonenamelast, 6) != 0) { + zonenamelast = zonename; + } + + // Populate the line + pgl = &pagelines[p]; + + pgl->line = l; + pgl->node = nodenum; + pgl->type = typename; + pgl->zone = zonename; + for (o = 0; o < pageorders_cnt; o++) + pgl->free_pages_size[o] = str2uint64_t(procfile_lineword(ff, l, o+6)) * 1 << o; + + p++; + } + + // Init the RRD graphs + + // Per-Order: sum of all node, zone, type Grouped by order + if (do_global != CONFIG_BOOLEAN_NO) { + st_order = rrdset_create_localhost( + "mem" + , "pagetype_global" + , NULL + , "pagetype" + , NULL + , "System orders available" + , "B" + , PLUGIN_PROC_NAME + , PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME + , NETDATA_CHART_PRIO_MEM_PAGEFRAG + , update_every + , RRDSET_TYPE_STACKED + ); + for (o = 0; o < pageorders_cnt; o++) { + char id[3+1]; + snprintfz(id, 3, "%lu", o); + + char name[20+1]; + dim_name(name, o, pagesize); + + systemorders[o].rd = rrddim_add(st_order, id, name, pagesize, 1, RRD_ALGORITHM_ABSOLUTE); + } + } + + + // Per-Numa Node & Zone & Type (full detail). Only if sum(line) > 0 + st_nodezonetype = callocz(pagelines_cnt, sizeof(RRDSET)); + for (p = 0; p < pagelines_cnt; p++) { + pgl = &pagelines[p]; + + // Skip invalid, refused or empty pagelines if not explicitely requested + if (!pgl + || do_detail == CONFIG_BOOLEAN_NO + || (do_detail == CONFIG_BOOLEAN_AUTO && pageline_total_count(pgl) == 0 && netdata_zero_metrics_enabled != CONFIG_BOOLEAN_YES)) + continue; + + // "pagetype Node" + NUMA-NodeId + ZoneName + TypeName + char setid[13+1+2+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME+1]; + snprintfz(setid, 13+1+2+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME, "pagetype_Node%d_%s_%s", pgl->node, pgl->zone, pgl->type); + + // Skip explicitely refused charts + if (simple_pattern_matches(filter_types, setid)) + continue; + + // "Node" + NUMA-NodeID + ZoneName + TypeName + char setname[4+1+MAX_ZONETYPE_NAME+1+MAX_PAGETYPE_NAME +1]; + snprintfz(setname, MAX_ZONETYPE_NAME + MAX_PAGETYPE_NAME, "Node %d %s %s", + pgl->node, pgl->zone, pgl->type); + + st_nodezonetype[p] = rrdset_create_localhost( + "mem" + , setid + , NULL + , "pagetype" + , NULL + , setname + , "B" + , PLUGIN_PROC_NAME + , PLUGIN_PROC_MODULE_PAGETYPEINFO_NAME + , NETDATA_CHART_PRIO_MEM_PAGEFRAG + 1 + p + , update_every + , RRDSET_TYPE_STACKED + ); + for (o = 0; o < pageorders_cnt; o++) { + char dimid[3+1]; + snprintfz(dimid, 3, "%lu", o); + char dimname[20+1]; + dim_name(dimname, o, pagesize); + + pgl->rd[o] = rrddim_add(st_nodezonetype[p], dimid, dimname, pagesize, 1, RRD_ALGORITHM_ABSOLUTE); + } + } + } + + // -------------------------------------------------------------------- + // Update pagelines + + // Process each line + p = 0; + for (l=4; l