From f0bc01ddbdfcb948bb06cf05662b0901f297e59a Mon Sep 17 00:00:00 2001 From: Vladimir Kobal Date: Tue, 23 Apr 2019 14:20:00 +0300 Subject: [PATCH] Add ioping plugin (#5725) * Add ioping plugin * Update the documentation and comments * Finalize installation of ioping * Fix find_processors variable * Fix static build * Move ioping to libexec directory * Move ioping to plugins.d directory * Fix file existence check --- .gitignore | 1 + README.md | 89 ++++---- build/subst.inc | 1 + collectors/Makefile.am | 1 + collectors/README.md | 1 + collectors/apps.plugin/apps_groups.conf | 1 + collectors/ioping.plugin/Makefile.am | 24 ++ collectors/ioping.plugin/README.md | 82 +++++++ collectors/ioping.plugin/ioping.conf | 40 ++++ collectors/ioping.plugin/ioping.plugin.in | 212 ++++++++++++++++++ collectors/plugins.d/README.md | 2 + configure.ac | 1 + docs/Add-more-charts-to-netdata.md | 5 +- docs/generator/buildyaml.sh | 1 + health/Makefile.am | 1 + health/health.d/ioping.conf | 13 ++ netdata-installer.sh | 6 + packaging/makeself/install-or-update.sh | 2 +- .../makeself/jobs/50-ioping-1.1.install.sh | 18 ++ web/gui/dashboard_info.js | 6 + 20 files changed, 461 insertions(+), 46 deletions(-) create mode 100644 collectors/ioping.plugin/Makefile.am create mode 100644 collectors/ioping.plugin/README.md create mode 100644 collectors/ioping.plugin/ioping.conf create mode 100755 collectors/ioping.plugin/ioping.plugin.in create mode 100644 health/health.d/ioping.conf create mode 100755 packaging/makeself/jobs/50-ioping-1.1.install.sh diff --git a/.gitignore b/.gitignore index 60a9f79f37..3bea6e0e33 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,7 @@ collectors/charts.d.plugin/charts.d.plugin collectors/node.d.plugin/node.d.plugin collectors/python.d.plugin/python.d.plugin collectors/fping.plugin/fping.plugin +collectors/ioping.plugin/ioping.plugin collectors/go.d.plugin # installer generated files diff --git a/README.md b/README.md index 1cedfcd08a..812b16a7bd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # netdata [![Build Status](https://travis-ci.com/netdata/netdata.svg?branch=master)](https://travis-ci.com/netdata/netdata) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2231/badge)](https://bestpractices.coreinfrastructure.org/projects/2231) [![License: GPL v3+](https://img.shields.io/badge/License-GPL%20v3%2B-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) [![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Freadme&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]() - + [![Code Climate](https://codeclimate.com/github/netdata/netdata/badges/gpa.svg)](https://codeclimate.com/github/netdata/netdata) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/a994873f30d045b9b4b83606c3eb3498)](https://www.codacy.com/app/netdata/netdata?utm_source=github.com&utm_medium=referral&utm_content=netdata/netdata&utm_campaign=Badge_Grade) [![LGTM C](https://img.shields.io/lgtm/grade/cpp/g/netdata/netdata.svg?logo=lgtm)](https://lgtm.com/projects/g/netdata/netdata/context:cpp) [![LGTM JS](https://img.shields.io/lgtm/grade/javascript/g/netdata/netdata.svg?logo=lgtm)](https://lgtm.com/projects/g/netdata/netdata/context:javascript) [![LGTM PYTHON](https://img.shields.io/lgtm/grade/python/g/netdata/netdata.svg?logo=lgtm)](https://lgtm.com/projects/g/netdata/netdata/context:python) --- @@ -10,12 +10,12 @@ Netdata provides **unparalleled insights**, **in real-time**, of everything happ _Netdata is **fast** and **efficient**, designed to permanently run on all systems (**physical** & **virtual** servers, **containers**, **IoT** devices), without disrupting their core function._ -Netdata is **free, open-source software** and it currently runs on **Linux**, **FreeBSD**, and **MacOS**. +Netdata is **free, open-source software** and it currently runs on **Linux**, **FreeBSD**, and **MacOS**. -![cncf](https://www.cncf.io/wp-content/uploads/2016/09/logo_cncf.png) +![cncf](https://www.cncf.io/wp-content/uploads/2016/09/logo_cncf.png) Netdata is in the [Cloud Native Computing Foundation (CNCF) landscape](https://landscape.cncf.io/format=card-mode&grouping=no&sort=stars) and it is the 3rd most starred open-source project. -Check the [CNCF TOC Netdata presentation](https://docs.google.com/presentation/d/18C8bCTbtgKDWqPa57GXIjB2PbjjpjsUNkLtZEz6YK8s/edit?usp=sharing). +Check the [CNCF TOC Netdata presentation](https://docs.google.com/presentation/d/18C8bCTbtgKDWqPa57GXIjB2PbjjpjsUNkLtZEz6YK8s/edit?usp=sharing). --- @@ -53,13 +53,13 @@ The following animated image, shows the top part of a typical netdata dashboard. *A typical netdata dashboard, in 1:1 timing. Charts can be panned by dragging them, zoomed in/out with `SHIFT` + `mouse wheel`, an area can be selected for zoom-in with `SHIFT` + `mouse selection`. Netdata is highly interactive and **real-time**, optimized to get the work done!* -> *We have a few online demos to experience it live: [https://my-netdata.io](https://my-netdata.io)* +> *We have a few online demos to experience it live: [https://my-netdata.io](https://my-netdata.io)* ## User base Netdata is used by hundreds of thousands of users all over the world. Check our [GitHub watchers list](https://github.com/netdata/netdata/watchers). -You will find people working for **Amazon**, **Atos**, **Baidu**, **Cisco Systems**, **Citrix**, **Deutsche Telekom**, **DigitalOcean**, +You will find people working for **Amazon**, **Atos**, **Baidu**, **Cisco Systems**, **Citrix**, **Deutsche Telekom**, **DigitalOcean**, **Elastic**, **EPAM Systems**, **Ericsson**, **Google**, **Groupon**, **Hortonworks**, **HP**, **Huawei**, **IBM**, **Microsoft**, **NewRelic**, **Nvidia**, **Red Hat**, **SAP**, **Selectel**, **TicketMaster**, **Vimeo**, and many more! @@ -72,9 +72,9 @@ We provide docker images for the most common architectures. These are statistics ### Registry When you install multiple netdata, they are integrated into **one distributed application**, via a [netdata registry](registry/#registry). This is a web browser feature and it allows us to count the number of unique users and unique netdata servers installed. The following information comes from the global public netdata registry we run: -[![User Base](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_entries&dimensions=persons&label=user%20base&units=M&value_color=blue&precision=2÷=1000000&v43)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) [![Monitored Servers](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_entries&dimensions=machines&label=servers%20monitored&units=k÷=1000&value_color=orange&precision=2&v43)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) [![Sessions Served](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_sessions&label=sessions%20served&units=M&value_color=yellowgreen&precision=2÷=1000000&v43)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) - -*in the last 24 hours:*
[![New Users Today](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_entries&dimensions=persons&after=-86400&options=unaligned&group=incremental-sum&label=new%20users%20today&units=null&value_color=blue&precision=0&v42)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) [![New Machines Today](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_entries&dimensions=machines&group=incremental-sum&after=-86400&options=unaligned&label=servers%20added%20today&units=null&value_color=orange&precision=0&v42)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) [![Sessions Today](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_sessions&after=-86400&group=incremental-sum&options=unaligned&label=sessions%20served%20today&units=null&value_color=yellowgreen&precision=0&v42)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) +[![User Base](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_entries&dimensions=persons&label=user%20base&units=M&value_color=blue&precision=2÷=1000000&v43)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) [![Monitored Servers](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_entries&dimensions=machines&label=servers%20monitored&units=k÷=1000&value_color=orange&precision=2&v43)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) [![Sessions Served](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_sessions&label=sessions%20served&units=M&value_color=yellowgreen&precision=2÷=1000000&v43)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) + +*in the last 24 hours:*
[![New Users Today](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_entries&dimensions=persons&after=-86400&options=unaligned&group=incremental-sum&label=new%20users%20today&units=null&value_color=blue&precision=0&v42)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) [![New Machines Today](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_entries&dimensions=machines&group=incremental-sum&after=-86400&options=unaligned&label=servers%20added%20today&units=null&value_color=orange&precision=0&v42)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) [![Sessions Today](https://registry.my-netdata.io/api/v1/badge.svg?chart=netdata.registry_sessions&after=-86400&group=incremental-sum&options=unaligned&label=sessions%20served%20today&units=null&value_color=yellowgreen&precision=0&v42)](https://registry.my-netdata.io/#menu_netdata_submenu_registry) ## Quick Start @@ -145,9 +145,9 @@ Netdata is **open-source**, **free**, super **fast**, very **easy**, completely It has been designed by **SysAdmins**, **DevOps** and **Developers** for troubleshooting performance problems, not just visualize metrics. -## News +## News -`Mar 14th, 2019` - **[netdata v1.13.0 released!](https://github.com/netdata/netdata/releases)** +`Mar 14th, 2019` - **[netdata v1.13.0 released!](https://github.com/netdata/netdata/releases)** Release 1.13.0 contains 14 bug fixes and 8 improvements. @@ -157,20 +157,20 @@ A [major refactoring of the python.d plugin](https://github.com/netdata/netdata/ We also added charts for IPC shared memory segments and total memory used. -`Feb 28th, 2019` - **[netdata v1.12.2 released!](https://github.com/netdata/netdata/releases)** +`Feb 28th, 2019` - **[netdata v1.12.2 released!](https://github.com/netdata/netdata/releases)** Patch release 1.12.2 contains 7 bug fixes and 4 improvements. The main motivation behind a new patch release is the introduction of a **stable release channel**. -A "stable" installation and update channel was always on our roadmap, but it became a necessity when we realized that our users in China could not use the nightly releases published on Google Cloud. The "stable" channel is based on our official GitHub releases and uses assets hosted on GitHub. +A "stable" installation and update channel was always on our roadmap, but it became a necessity when we realized that our users in China could not use the nightly releases published on Google Cloud. The "stable" channel is based on our official GitHub releases and uses assets hosted on GitHub. -We are also introducing a new **Oracle DB collector** module, implemented in Python. +We are also introducing a new **Oracle DB collector** module, implemented in Python. -`Feb 21st, 2019` - **[netdata v1.12.1 released!](https://github.com/netdata/netdata/releases)** +`Feb 21st, 2019` - **[netdata v1.12.1 released!](https://github.com/netdata/netdata/releases)** Patch release 1.12.1 contains 22 bug fixes and 8 improvements. -`Feb 14th, 2019` - **[netdata v1.12.0 released!](https://github.com/netdata/netdata/releases)** +`Feb 14th, 2019` - **[netdata v1.12.0 released!](https://github.com/netdata/netdata/releases)** Release 1.12 is made out of 211 pull requests and 22 bug fixes. The key improvements are: @@ -182,22 +182,22 @@ The key improvements are: - Bind different functions of the netdata APIs to different ports - Improved installation and updates -`Nov 22nd, 2018` - **[netdata v1.11.1 released!](https://github.com/netdata/netdata/releases)** +`Nov 22nd, 2018` - **[netdata v1.11.1 released!](https://github.com/netdata/netdata/releases)** - Improved internal database to support values above 64bit. - New data collection plugins: [`openldap`](collectors/python.d.plugin/openldap/), [`tor`](collectors/python.d.plugin/tor/), [`nvidia_smi`](collectors/python.d.plugin/nvidia_smi/). - Improved data collection plugins: netdata now supports monitoring network interface aliases, [`smartd_log`](collectors/python.d.plugin/smartd_log/), [`cpufreq`](collectors/proc.plugin/README.md#cpu-frequency), [`sensors`](collectors/python.d.plugin/sensors/). - Health monitoring improvements: network interface congestion alarm restored, [`alerta.io`](health/notifications/alerta/), `conntrack_max`. -- `my-netdata`menu has been refactored. +- `my-netdata`menu has been refactored. - Packaging: `openrc` service definition got a few improvements. ---- - -`Sep 18, 2018` - **netdata has its own organization** - -Netdata used to be a [firehol.org](https://firehol.org) project, accessible as `firehol/netdata`. - -Netdata now has its own github organization `netdata`, so all github URLs are now `netdata/netdata`. The old github URLs, repo clones, forks, etc redirect automatically to the new repo. +--- + +`Sep 18, 2018` - **netdata has its own organization** + +Netdata used to be a [firehol.org](https://firehol.org) project, accessible as `firehol/netdata`. + +Netdata now has its own github organization `netdata`, so all github URLs are now `netdata/netdata`. The old github URLs, repo clones, forks, etc redirect automatically to the new repo. ## How it works @@ -218,12 +218,12 @@ Function|Description|Documentation The result is a highly efficient, low latency system, supporting multiple readers and one writer on each metric. -## Infographic - -This is a high level overview of netdata feature set and architecture. -Click it to to interact with it (it has direct links to documentation). - -[![image](https://user-images.githubusercontent.com/2662304/47672043-a47eb480-dbb9-11e8-92a4-fa422d053309.png)](https://my-netdata.io/infographic.html) +## Infographic + +This is a high level overview of netdata feature set and architecture. +Click it to to interact with it (it has direct links to documentation). + +[![image](https://user-images.githubusercontent.com/2662304/47672043-a47eb480-dbb9-11e8-92a4-fa422d053309.png)](https://my-netdata.io/infographic.html) ## Features @@ -240,7 +240,7 @@ This is what you should expect from Netdata: - **Zero disk I/O** - while it runs, it does not load or save anything (except `error` and `access` logs). - **Zero configuration** - auto-detects everything, it can collect up to 10000 metrics per server out of the box. - **Zero maintenance** - You just run it, it does the rest. -- **Zero dependencies** - it is even its own web server, for its static web files and its web API (though its plugins may require additional libraries, depending on the applications monitored). +- **Zero dependencies** - it is even its own web server, for its static web files and its web API (though its plugins may require additional libraries, depending on the applications monitored). - **Scales to infinity** - you can install it on all your servers, containers, VMs and IoTs. Metrics are not centralized by default, so there is no limit. - **Several operating modes** - Autonomous host monitoring (the default), headless data collector, forwarding proxy, store and forward proxy, central multi-host monitoring, in all possible configurations. Each node may have different metrics retention policy and run with or without health monitoring. @@ -297,7 +297,7 @@ To improve visual anomaly detection across charts, the user can highlight a time > Highlighted ranges are propagated from netdata server to netdata server, when navigating via the [`my-netdata` menu](registry#registry). -## What does it monitor +## What does it monitor Netdata data collection is **extensible** - you can monitor anything you can get a metric for. Its [Plugin API](collectors/plugins.d/) supports all programing languages (anything can be a netdata plugin, BASH, python, perl, node.js, java, Go, ruby, etc). @@ -336,7 +336,7 @@ Its [Plugin API](collectors/plugins.d/) supports all programing languages (anyth - **[slab](collectors/proc.plugin/)** - collects info about the Linux kernel memory usage. #### Disks -- **[block devices](collectors/proc.plugin/)** - per disk: I/O, operations, backlog, utilization, space, etc. +- **[block devices](collectors/proc.plugin/)** - per disk: I/O, operations, backlog, utilization, space, etc. - **[BCACHE](collectors/proc.plugin/)** - detailed performance of SSD caching devices. - **[DiskSpace](collectors/proc.plugin/)** - monitors disk space usage. - **[mdstat](collectors/proc.plugin/)** - software RAID. @@ -346,11 +346,12 @@ Its [Plugin API](collectors/plugins.d/) supports all programing languages (anyth - **[Veritas Volume Manager](collectors/proc.plugin/)** - naming disks. - **[megacli](collectors/python.d.plugin/megacli/)** - adapter, physical drives and battery stats. - **[adaptec_raid](collectors/python.d.plugin/adaptec_raid/)** - logical and physical devices health metrics. +- **[ioping](collectors/ioping.plugin/)** - to measure disk read/write latency. #### Filesystems - **[BTRFS](collectors/proc.plugin/)** - detailed disk space allocation and usage. - **[Ceph](collectors/python.d.plugin/ceph/)** - OSD usage, Pool usage, number of objects, etc. -- **[NFS file servers and clients](collectors/proc.plugin/)** - NFS v2, v3, v4: I/O, cache, read ahead, RPC calls +- **[NFS file servers and clients](collectors/proc.plugin/)** - NFS v2, v3, v4: I/O, cache, read ahead, RPC calls - **[Samba](collectors/python.d.plugin/samba/)** - performance metrics of Samba SMB2 file sharing. - **[ZFS](collectors/proc.plugin/)** - detailed performance and resource usage. @@ -359,7 +360,7 @@ Its [Plugin API](collectors/plugins.d/) supports all programing languages (anyth - **[Netfilter](collectors/proc.plugin/)** - everything about the netfilter connection tracker. - **[SynProxy](collectors/proc.plugin/)** - collects performance data about the linux SYNPROXY (DDoS). - **[NFacct](collectors/nfacct.plugin/)** - collects accounting data from iptables. -- **[Network QoS](collectors/tc.plugin/)** - the only tool that visualizes network `tc` classes in real-time +- **[Network QoS](collectors/tc.plugin/)** - the only tool that visualizes network `tc` classes in real-time - **[FPing](collectors/fping.plugin/)** - to measure latency and packet loss between any number of hosts. - **[ISC dhcpd](collectors/python.d.plugin/isc_dhcpd/)** - pools utilization, leases, etc. - **[AP](collectors/charts.d.plugin/ap/)** - collects Linux access point performance data (`hostapd`). @@ -373,7 +374,7 @@ Its [Plugin API](collectors/plugins.d/) supports all programing languages (anyth #### Processes - **[System Processes](collectors/proc.plugin/)** - running, blocked, forks, active. -- **[Applications](collectors/apps.plugin/)** - by grouping the process tree and reporting CPU, memory, disk reads, disk writes, swap, threads, pipes, sockets - per process group. +- **[Applications](collectors/apps.plugin/)** - by grouping the process tree and reporting CPU, memory, disk reads, disk writes, swap, threads, pipes, sockets - per process group. - **[systemd](collectors/cgroups.plugin/)** - monitors systemd services using CGROUPS. #### Users @@ -430,7 +431,7 @@ Its [Plugin API](collectors/plugins.d/) supports all programing languages (anyth - **[dns_query_time](collectors/python.d.plugin/dns_query_time/)** - DNS query time statistics. #### Time Servers -- **[chrony](collectors/python.d.plugin/chrony/)** - uses the `chronyc` command to collect chrony statistics (Frequency, Last offset, RMS offset, Residual freq, Root delay, Root dispersion, Skew, System time). +- **[chrony](collectors/python.d.plugin/chrony/)** - uses the `chronyc` command to collect chrony statistics (Frequency, Last offset, RMS offset, Residual freq, Root delay, Root dispersion, Skew, System time). - **[ntpd](collectors/python.d.plugin/ntpd/)** - connects to multiple ntpd servers (local or remote) to provide statistics of system variables and optional also peer variables. #### Mail Servers @@ -482,10 +483,10 @@ Its [Plugin API](collectors/plugins.d/) supports all programing languages (anyth #### Provisioning Systems - **[Puppet](collectors/python.d.plugin/puppet/)** - connects to multiple Puppet Server and Puppet DB instances (local or remote) to collect real-time status metrics. -You can easily extend Netdata, by writing plugins that collect data from any source, using any computer language. - ---- - +You can easily extend Netdata, by writing plugins that collect data from any source, using any computer language. + +--- + ## Documentation The netdata documentation is at [https://docs.netdata.cloud](https://docs.netdata.cloud). But you can also find it inside the repo, so by just navigating the repo on github you can find all the documentation. @@ -523,8 +524,8 @@ You can also find netdata on: - [StackShare](https://stackshare.io/netdata) ## License - -netdata is [GPLv3+](LICENSE). + +netdata is [GPLv3+](LICENSE). Netdata re-distributes other open-source tools and libraries. Please check the [third party licenses](REDISTRIBUTED.md). diff --git a/build/subst.inc b/build/subst.inc index 558d33adf9..db891f24c1 100644 --- a/build/subst.inc +++ b/build/subst.inc @@ -2,6 +2,7 @@ if sed \ -e 's#[@]localstatedir_POST@#$(localstatedir)#g' \ -e 's#[@]sbindir_POST@#$(sbindir)#g' \ + -e 's#[@]pluginsdir_POST@#$(pluginsdir)#g' \ -e 's#[@]configdir_POST@#$(configdir)#g' \ -e 's#[@]libconfigdir_POST@#$(libconfigdir)#g' \ -e 's#[@]cachedir_POST@#$(cachedir)#g' \ diff --git a/collectors/Makefile.am b/collectors/Makefile.am index 9f07fd4dc9..87a037e762 100644 --- a/collectors/Makefile.am +++ b/collectors/Makefile.am @@ -11,6 +11,7 @@ SUBDIRS = \ cups.plugin \ diskspace.plugin \ fping.plugin \ + ioping.plugin \ freebsd.plugin \ freeipmi.plugin \ idlejitter.plugin \ diff --git a/collectors/README.md b/collectors/README.md index 86c2a9fd24..154d193e49 100644 --- a/collectors/README.md +++ b/collectors/README.md @@ -30,6 +30,7 @@ plugin|lang|O/S|runs as|modular|description [cups.plugin](cups.plugin/)|`C`|any|external|-|monitors **CUPS** [diskspace.plugin](diskspace.plugin/)|`C`|linux|internal|-|collects disk space usage metrics on Linux mount points [fping.plugin](fping.plugin/)|`C`|any|external|-|measures network latency, jitter and packet loss between the monitored node and any number of remote network end points. +[ioping.plugin](ioping.plugin/)|`C`|any|external|-|measures disk read/write latency. [freebsd.plugin](freebsd.plugin/)|`C`|freebsd|internal|yes|collects resource usage and performance data on FreeBSD systems [freeipmi.plugin](freeipmi.plugin/)|`C`|linux, freebsd|external|-|collects metrics from enterprise hardware sensors, on Linux and FreeBSD servers. [idlejitter.plugin](idlejitter.plugin/)|`C`|any|internal|-|measures CPU latency and jitter on all operating systems diff --git a/collectors/apps.plugin/apps_groups.conf b/collectors/apps.plugin/apps_groups.conf index da71043298..24e0f29bb1 100644 --- a/collectors/apps.plugin/apps_groups.conf +++ b/collectors/apps.plugin/apps_groups.conf @@ -82,6 +82,7 @@ node.d.plugin: *node.d.plugin* python.d.plugin: *python.d.plugin* tc-qos-helper: *tc-qos-helper.sh* fping: fping +ioping: ioping go.d.plugin: *go.d.plugin* # ----------------------------------------------------------------------------- diff --git a/collectors/ioping.plugin/Makefile.am b/collectors/ioping.plugin/Makefile.am new file mode 100644 index 0000000000..7610b0e077 --- /dev/null +++ b/collectors/ioping.plugin/Makefile.am @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +CLEANFILES = \ + ioping.plugin \ + $(NULL) + +include $(top_srcdir)/build/subst.inc +SUFFIXES = .in + +dist_plugins_SCRIPTS = \ + ioping.plugin \ + $(NULL) + +dist_noinst_DATA = \ + ioping.plugin.in \ + README.md \ + $(NULL) + +dist_libconfig_DATA = \ + ioping.conf \ + $(NULL) diff --git a/collectors/ioping.plugin/README.md b/collectors/ioping.plugin/README.md new file mode 100644 index 0000000000..15fdf6ff7b --- /dev/null +++ b/collectors/ioping.plugin/README.md @@ -0,0 +1,82 @@ +# ioping.plugin + +The ioping plugin supports monitoring latency for any number of directories/files/devices, +by pinging them with `ioping`. + +A recent version of `ioping` is required (one that supports option ` -N `). +The supplied plugin can install it, by running: + +```sh +/usr/libexec/netdata/plugins.d/ioping.plugin install +``` + +The `-e` option can be supplied to indicate where the netdata environment file is installed. The default path is `/etc/netdata/.environment`. + +The above will download, build and install the right version as `/usr/libexec/netdata/plugins.d/ioping`. + +Then you need to edit `/etc/netdata/ioping.conf` (to edit it on your system run +`/etc/netdata/edit-config ioping.conf`) like this: + +```sh +# uncomment the following line - it should already be there +ioping="/usr/libexec/netdata/plugins.d/ioping" + +# set here the directory/file/device, you need to ping +destination="destination" + +# override the chart update frequency - the default is inherited from netdata +update_every="1s" + +# the request size in bytes to ping the destination +request_size="4k" + +# other iping options - these are the defaults +ioping_opts="-T 1000000 -R" +``` + +## alarms + +netdata will automatically attach a few alarms for each host. +Check the [latest versions of the ioping alarms](../../health/health.d/ioping.conf) + +## Multiple ioping Plugins With Different Settings + +You may need to run multiple ioping plugins with different settings or different end points. +For example, you may need to ping one destination once per 10 seconds, and another once per second. + +netdata allows you to add as many `ioping` plugins as you like. + +Follow this procedure: + +**1. Create New ioping Configuration File** + + +```sh +# Step Into Configuration Directory +cd /etc/netdata + +# Copy Original ioping Configuration File To New Configuration File +cp ioping.conf ioping2.conf +``` + +Edit `ioping2.conf` and set the settings and the destination you need for the seconds instance. + +**2. Soft Link Original ioping Plugin to New Plugin File** + +```sh +# Become root (If The Step Step Is Performed As Non-Root User) +sudo su + +# Step Into The Plugins Directory +cd /usr/libexec/netdata/plugins.d + +# Link ioping.plugin to ioping2.plugin +ln -s ioping.plugin ioping2.plugin +``` + +That's it. netdata will detect the new plugin and start it. + +You can name the new plugin any name you like. +Just make sure the plugin and the configuration file have the same name. + +[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fcollectors%2Fioping.plugin%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]() diff --git a/collectors/ioping.plugin/ioping.conf b/collectors/ioping.plugin/ioping.conf new file mode 100644 index 0000000000..86f0de7f4a --- /dev/null +++ b/collectors/ioping.plugin/ioping.conf @@ -0,0 +1,40 @@ +# no need for shebang - this file is sourced from ioping.plugin + +# ioping.plugin requires a recent version of ioping. +# +# You can get it on your system, by running: +# +# /usr/libexec/netdata/plugins.d/ioping.plugin install + +# ----------------------------------------------------------------------------- +# configuration options + +# The ioping binary to use. We need one that can output netdata friendly info +# (supporting: -N). If you have multiple versions, put here the full filename +# of the right one + +#ioping="/usr/libexec/netdata/plugins.d/ioping" + + +# The directory/file/device to ioping + +destination="" + + +# The update frequency of the chart in seconds (symbolic modifiers are supported) +# the default is inherited from netdata + +#update_every="1s" + + +# The request size in bytes to ioping the destination (symbolic modifiers are supported) +# by default 4k chunks are used + +#request_size="4k" + + +# Other ioping options +# the defaults: +# -T 1000000 = maximum valid request time (us) + +#ioping_opts="-T 1000000" diff --git a/collectors/ioping.plugin/ioping.plugin.in b/collectors/ioping.plugin/ioping.plugin.in new file mode 100755 index 0000000000..9f9babd89c --- /dev/null +++ b/collectors/ioping.plugin/ioping.plugin.in @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-3.0-or-later + +# netdata +# real-time performance and health monitoring, done right! +# (C) 2017 Costa Tsaousis +# GPL v3+ +# +# This plugin requires a latest version of ioping. +# You can compile it from source, by running me with option: install + +export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin" +export LC_ALL=C + +usage="$(basename "$0") [install] [-h] [-e] + +where: + install install ioping binary + -e, --env path to environment file (defauls to '/etc/netdata/.environment' + -h show this help text" + +INSTALL=0 +ENVIRONMENT_FILE="/etc/netdata/.environment" + +while :; do + case "$1" in + -h | --help) + echo "$usage" >&2 + exit 1 + ;; + install) + INSTALL=1 + shift + ;; + -e | --env) + ENVIRONMENT_FILE="$2" + shift 2 + ;; + -*) + echo "$usage" >&2 + exit 1 + ;; + *) break ;; + esac +done + +if [ "$INSTALL" == "1" ] + then + [ "${UID}" != 0 ] && echo >&2 "Please run me as root. This will install a single binary file: /usr/libexec/netdata/plugins.d/ioping." && exit 1 + + source "${ENVIRONMENT_FILE}" || exit 1 + + run() { + printf >&2 " > " + printf >&2 "%q " "${@}" + printf >&2 "\n" + "${@}" || exit 1 + } + + download() { + local git="$(which git 2>/dev/null || command -v git 2>/dev/null)" + [ ! -z "${git}" ] && run git clone "${1}" "${2}" && return 0 + + echo >&2 "Cannot find 'git' in this system." && exit 1 + } + + tmp=$(mktemp -d /tmp/netdata-ioping-XXXXXX) + [ ! -d "${NETDATA_PREFIX}/usr/libexec/netdata" ] && run mkdir -p "${NETDATA_PREFIX}/usr/libexec/netdata" + + run cd "${tmp}" + + if [ -d ioping-netdata ] + then + run rm -rf ioping-netdata || exit 1 + fi + + download 'https://github.com/netdata/ioping.git' 'ioping-netdata' + [ $? -ne 0 ] && exit 1 + run cd ioping-netdata || exit 1 + + INSTALL_PATH="${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/ioping" + + run make clean + run make + run mv ioping "${INSTALL_PATH}" + run chown root:"${NETDATA_GROUP}" "${INSTALL_PATH}" + run chmod 4750 "${INSTALL_PATH}" + echo >&2 + echo >&2 "All done, you have a compatible ioping now at ${INSTALL_PATH}." + echo >&2 + + exit 0 +fi + +# ----------------------------------------------------------------------------- + +PROGRAM_NAME="$(basename "${0}")" + +logdate() { + date "+%Y-%m-%d %H:%M:%S" +} + +log() { + local status="${1}" + shift + + echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}" + +} + +warning() { + log WARNING "${@}" +} + +error() { + log ERROR "${@}" +} + +info() { + log INFO "${@}" +} + +fatal() { + log FATAL "${@}" + echo "DISABLE" + exit 1 +} + +debug=0 +debug() { + [ $debug -eq 1 ] && log DEBUG "${@}" +} + +# ----------------------------------------------------------------------------- + +# store in ${plugin} the name we run under +# this allows us to copy/link ioping.plugin under a different name +# to have multiple ioping plugins running with different settings +plugin="${PROGRAM_NAME/.plugin/}" + + +# ----------------------------------------------------------------------------- + +# the frequency to send info to netdata +# passed by netdata as the first parameter +update_every="${1-1}" + +# the netdata configuration directory +# passed by netdata as an environment variable +[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@" +[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@" + +# the netdata directory for internal binaries +[ -z "${NETDATA_PLUGINS_DIR}" ] && NETDATA_PLUGINS_DIR="@pluginsdir_POST@" + +# ----------------------------------------------------------------------------- +# configuration options +# can be overwritten at /etc/netdata/ioping.conf + +# the ioping binary to use +# we need one that can output netdata friendly info (supporting: -N) +# if you have multiple versions, put here the full filename of the right one +ioping="${NETDATA_PLUGINS_DIR}/ioping" + +# the destination to ioping +destination="" + +# the request size in bytes to ping the disk +request_size="4k" + +# ioping options +ioping_opts="-T 1000000" + +# ----------------------------------------------------------------------------- +# load the configuration files + +for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/${plugin}.conf" "${NETDATA_USER_CONFIG_DIR}/${plugin}.conf" +do + if [ -f "${CONFIG}" ] + then + info "Loading config file '${CONFIG}'..." + source "${CONFIG}" + [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'." + else + warning "Cannot find file '${CONFIG}'." + fi +done + +if [ -z "${destination}" ] +then + fatal "destination is not configured - nothing to do." +fi + +if [ ! -f "${ioping}" ] +then + fatal "ioping command is not found. Please set its full path in '${NETDATA_USER_CONFIG_DIR}/${plugin}.conf'" +fi + +if [ ! -x "${ioping}" ] +then + fatal "ioping command '${ioping}' is not executable - cannot proceed." +fi + +# the ioping options we will use +options=( -N -i ${update_every} -s ${request_size} ${ioping_opts} ${destination} ) + +# execute ioping +info "starting ioping: ${ioping} ${options[*]}" +exec "${ioping}" "${options[@]}" + +# if we cannot execute ioping, stop +fatal "command '${ioping} ${options[*]}' failed to be executed (returned code $?)." diff --git a/collectors/plugins.d/README.md b/collectors/plugins.d/README.md index d5b9b85ef8..105a60eb57 100644 --- a/collectors/plugins.d/README.md +++ b/collectors/plugins.d/README.md @@ -11,6 +11,7 @@ plugin|language|O/S|description [charts.d.plugin](../charts.d.plugin/)|`BASH`|all|a **plugin orchestrator** for data collection modules written in `BASH` v4+. [cups.plugin](../cups.plugin/)|`C`|all|monitors **CUPS** [fping.plugin](../fping.plugin/)|`C`|all|measures network latency, jitter and packet loss between the monitored node and any number of remote network end points. +[ioping.plugin](../ioping.plugin/)|`C`|all|measures disk latency. [freeipmi.plugin](../freeipmi.plugin/)|`C`|linux|collects metrics from enterprise hardware sensors, on Linux servers. [nfacct.plugin](../nfacct.plugin/)|`C`|linux|collects netfilter firewall, connection tracker and accounting metrics using `libmnl` and `libnetfilter_acct`. [xenstat.plugin](../xenstat.plugin/)|`C`|linux|collects XenServer and XCP-ng metrics using `lxenstat`. @@ -64,6 +65,7 @@ Example: # charts.d = yes # fping = yes + # ioping = yes # node.d = yes # python.d = yes ``` diff --git a/configure.ac b/configure.ac index d663bcdcbd..565f05c133 100644 --- a/configure.ac +++ b/configure.ac @@ -692,6 +692,7 @@ AC_CONFIG_FILES([ collectors/checks.plugin/Makefile collectors/diskspace.plugin/Makefile collectors/fping.plugin/Makefile + collectors/ioping.plugin/Makefile collectors/freebsd.plugin/Makefile collectors/freeipmi.plugin/Makefile collectors/cups.plugin/Makefile diff --git a/docs/Add-more-charts-to-netdata.md b/docs/Add-more-charts-to-netdata.md index b62322f29b..382cd8d3ac 100644 --- a/docs/Add-more-charts-to-netdata.md +++ b/docs/Add-more-charts-to-netdata.md @@ -65,6 +65,7 @@ To control which plugins netdata run, edit `netdata.conf` and check the `[plugin # node.d = yes # python.d = yes # fping = yes + # ioping = yes # charts.d = yes # apps = yes # xenstat = yes @@ -94,7 +95,7 @@ sudo su -s /bin/bash netdata ``` Similarly, you can use `charts.d.plugin` for BASH plugins and `node.d.plugin` for node.js plugins. -Other plugins (like `apps.plugin`, `freeipmi.plugin`, `fping.plugin`) use the native netdata plugin API and can be run directly. +Other plugins (like `apps.plugin`, `freeipmi.plugin`, `fping.plugin`, `ioping.plugin`) use the native netdata plugin API and can be run directly. If you need to configure a netdata plugin or module, all user supplied configuration is kept at `/etc/netdata` while the stock versions of all files is at `/usr/lib/netdata/conf.d`. To copy a stock file and edit it, run `/etc/netdata/edit-config`. Running this command without an argument, will list the available stock files. @@ -113,6 +114,7 @@ plugin | language | plugin
configuration | modules
configuration | `charts.d.plugin`
(external plugin orchestrator for BASH modules)|`BASH`|`charts.d.conf`|a file for each module in `/etc/netdata/charts.d/` `diskspace.plugin`
(internal plugin for collecting Linux mount points usage)|`C`|`netdata.conf` section `[plugin:diskspace]`|N/A `fping.plugin`
(external plugin for collecting network latencies)|`C`|`fping.conf`|This plugin is a wrapper for the `fping` command. +`ioping.plugin`
(external plugin for collecting disk latencies)|`C`|`ioping.conf`|This plugin is a wrapper for the `ioping` command. `freeipmi.plugin`
(external plugin for collecting IPMI h/w sensors)|`C`|`netdata.conf` section `[plugin:freeipmi]` `idlejitter.plugin`
(internal plugin for monitoring CPU jitter)|`C`|N/A|N/A `macos.plugin`
(internal plugin for monitoring MacOS system resources)|`C`|`netdata.conf` section `[plugin:macos]`|one section for each module `[plugin:macos:MODULE]`. Each module may provide additional sections in the form of `[plugin:macos:MODULE:SUBSECTION]`. @@ -318,6 +320,7 @@ xenstat|C|Collects host and domain statistics for XenServer or XCP-ng hypervisor application|language|notes| :---------:|:------:|:----| apps|C|`apps.plugin` collects resource usage statistics for all processes running in the system. It groups the entire process tree and reports dozens of metrics for CPU utilization, memory footprint, disk I/O, swap memory, network connections, open files and sockets, etc. It reports metrics for application groups, users and user groups.
 
[Documentation of `apps.plugin`](../collectors/apps.plugin/).
 
netdata plugin: [`apps_plugin.c`](../collectors/apps.plugin)
configuration file: [`apps_groups.conf`](../collectors/apps.plugin)| +ioping|C|Charts disk latency statistics for a directory/file/device, using the `ioping` command. A recent (probably unreleased) version of ioping is required. The plugin supplied can install it in `/usr/local`.
 
netdata plugin: [ioping.plugin](../collectors/ioping.plugin) (this is a shell wrapper to start ioping - once ioping is started, netdata and ioping communicate directly - it can also install the right version of ioping)
configuration file: [ioping.conf](../collectors/ioping.plugin)| cpu_apps|BASH
Shell Script|Collects the CPU utilization of select apps.

DEPRECATED IN FAVOR OF `apps.plugin`. It is still supplied only as an example module to shell scripting plugins.
 
netdata plugin: [charts.d.plugin](../collectors/charts.d.plugin#chartsdplugin)
plugin module: [cpu_apps.chart.sh](../collectors/charts.d.plugin/cpu_apps)
configuration file: [charts.d/cpu_apps.conf](../collectors/charts.d.plugin/cpu_apps)| load_average|BASH
Shell Script|Collects the current system load average.

DEPRECATED IN FAVOR OF THE NETDATA INTERNAL ONE. It is still supplied only as an example module to shell scripting plugins.
 
netdata plugin: [charts.d.plugin](../collectors/charts.d.plugin#chartsdplugin)
plugin module: [load_average.chart.sh](../collectors/charts.d.plugin/load_average)
configuration file: [charts.d/load_average.conf](../collectors/charts.d.plugin/load_average)| mem_apps|BASH
Shell Script|Collects the memory footprint of select applications.

DEPRECATED IN FAVOR OF `apps.plugin`. It is still supplied only as an example module to shell scripting plugins.
 
netdata plugin: [charts.d.plugin](../collectors/charts.d.plugin#chartsdplugin)
plugin module: [mem_apps.chart.sh](../collectors/charts.d.plugin/mem_apps)
configuration file: [charts.d/mem_apps.conf](../collectors/charts.d.plugin/mem_apps)| diff --git a/docs/generator/buildyaml.sh b/docs/generator/buildyaml.sh index d905be2ed3..a8a8f4f9ed 100755 --- a/docs/generator/buildyaml.sh +++ b/docs/generator/buildyaml.sh @@ -224,6 +224,7 @@ echo -ne " - BASH: navpart 3 collectors/apps.plugin navpart 3 collectors/cups.plugin navpart 3 collectors/fping.plugin +navpart 3 collectors/ioping.plugin navpart 3 collectors/freeipmi.plugin navpart 3 collectors/nfacct.plugin navpart 3 collectors/xenstat.plugin diff --git a/health/Makefile.am b/health/Makefile.am index 0180111880..62a4c6d333 100644 --- a/health/Makefile.am +++ b/health/Makefile.am @@ -39,6 +39,7 @@ dist_healthconfig_DATA = \ health.d/elasticsearch.conf \ health.d/entropy.conf \ health.d/fping.conf \ + health.d/ioping.conf \ health.d/fronius.conf \ health.d/haproxy.conf \ health.d/httpcheck.conf \ diff --git a/health/health.d/ioping.conf b/health/health.d/ioping.conf new file mode 100644 index 0000000000..59a5c8edc3 --- /dev/null +++ b/health/health.d/ioping.conf @@ -0,0 +1,13 @@ +template: disk_latency +families: * + on: ioping.latency + lookup: average -10s unaligned of average + units: ms + every: 10s + green: 500 + red: 1000 + warn: $this > $green OR $max > $red + crit: $this > $red + info: average round trip delay during the last 10 seconds + delay: down 30m multiplier 1.5 max 2h + to: sysadmin diff --git a/netdata-installer.sh b/netdata-installer.sh index 6fb20287d7..d07fb6bff5 100755 --- a/netdata-installer.sh +++ b/netdata-installer.sh @@ -718,6 +718,11 @@ if [ "${UID}" -eq 0 ]; then run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/xenstat.plugin" fi + if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/ioping" ]; then + run chown root:${NETDATA_GROUP} "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/ioping" + run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/ioping" + fi + if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/cgroup-network" ]; then run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/cgroup-network" run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/cgroup-network" @@ -989,6 +994,7 @@ NETDATA_PREFIX="${NETDATA_PREFIX}" NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS}" NETDATA_ADDED_TO_GROUPS="${NETDATA_ADDED_TO_GROUPS}" INSTALL_UID="${UID}" +NETDATA_GROUP="${NETDATA_GROUP}" REINSTALL_COMMAND="${REINSTALL_COMMAND}" RELEASE_CHANNEL="${RELEASE_CHANNEL}" # This value is meant to be populated by autoupdater (if enabled) diff --git a/packaging/makeself/install-or-update.sh b/packaging/makeself/install-or-update.sh index 2ab50a17cf..fc4e6d077d 100755 --- a/packaging/makeself/install-or-update.sh +++ b/packaging/makeself/install-or-update.sh @@ -195,7 +195,7 @@ run chown -R ${NETDATA_USER}:${NETDATA_GROUP} /opt/netdata progress "fix plugin permissions" -for x in apps.plugin freeipmi.plugin cgroup-network +for x in apps.plugin freeipmi.plugin ioping cgroup-network do f="usr/libexec/netdata/plugins.d/${x}" diff --git a/packaging/makeself/jobs/50-ioping-1.1.install.sh b/packaging/makeself/jobs/50-ioping-1.1.install.sh new file mode 100755 index 0000000000..83c778c158 --- /dev/null +++ b/packaging/makeself/jobs/50-ioping-1.1.install.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-3.0-or-later + +. $(dirname "${0}")/../functions.sh "${@}" || exit 1 + +fetch "netdata-ioping-43d15a5" "https://github.com/netdata/ioping/tarball/master" + +export CFLAGS="-static" + +run make clean +run make -j$(find_processors) +run mkdir -p ${NETDATA_INSTALL_PATH}/usr/libexec/netdata/plugins.d/ +run install -o root -g root -m 4750 ioping ${NETDATA_INSTALL_PATH}/usr/libexec/netdata/plugins.d/ + +if [ ${NETDATA_BUILD_WITH_DEBUG} -eq 0 ] +then + run strip ${NETDATA_INSTALL_PATH}/usr/libexec/netdata/plugins.d/ioping +fi diff --git a/web/gui/dashboard_info.js b/web/gui/dashboard_info.js index 99078ff4ca..aab11ba347 100644 --- a/web/gui/dashboard_info.js +++ b/web/gui/dashboard_info.js @@ -213,6 +213,12 @@ netdataDashboard.menu = { info: 'Network latency statistics, via fping. fping is a program to send ICMP echo probes to network hosts, similar to ping, but much better performing when pinging multiple hosts. fping versions after 3.15 can be directly used as netdata plugins.' }, + 'ioping': { + title: 'ioping', + icon: '', + info: 'Disk latency statistics, via ioping. ioping is a program to read/write data probes from/to a disk.' + }, + 'httpcheck': { title: 'Http Check', icon: '',