add Opensearch\Elasticsearch monitoring (#14053)

* add new poller

* add a missing ;

* formatting cleanup

* graph stuff and metrics move

* add rrd name

* clean up metrics/rrd def

* more metric/rrd def cleanup

* cleanup

* add basic opensearch graphs

* add opensearch to apps.inc.php

* begin work on opensearch app page

* formatting cleanup

* add translog graphs

* add a missing graph

* fix pending tasks

* add the ability to fetch the saved cluster name

* add fetching the cluster name

* correct the opensearch comment

* add combined shard stats

* add indexing graphs

* correct graph name

* correct some units as being per second

* add more graphs

* add more items for graph sets

* cleanup of units and naming... also more graphs

* more graph stuff

* change the RRD def again and define a few more graphs

* finish basic graph sets

* more graph stuff

* another rrd def change

* add more graphs

* add some more graph sets

* correct unit for c_task_max_in_time

* more graph stuff

* more graph stuff

* correct the unit

* add missing tw_time and another rrd def change

* another unit change

* add trc graphs

* more graph stuff

* add tseg graphs

* add all shards graph to both cluster items

* more graph stuff

* update opensearch app page

* add Opensearch\Elasticsearch app

* add opensearch tests

* run php-cs-fixer on two files

* add alert examples for checking cluster status

* remove an item that was accidentally added as a metric in the test but is not

* derp! thanks jellyfrog

* make it come up as Elisticsearch\Opensearch in the webui

* no longer use components, but app_data, for cluster name change

* update the web side for opensearch for using app_data

* style fix

* update opensearch for new app data stuff

* update to the new Application model

* update poller and device app page for ES/OS

* style cleanup

* update graphs

* test fix

* more test cleanup

* Update alert_rules.json

* begin work on breaking out the RRDs

* update all non-multi rrd graphs for opensearch

* update time_all

* add a unass shards graph

* correct rrd name

* should all be good now

* add missing tm stats

* Un Assigned -> Unassigned

* style cleanup

* another style fix

* remove cluster_name from saved metrics as it is not a metric

Co-authored-by: Tony Murray <murraytony@gmail.com>
Co-authored-by: Jellyfrog <Jellyfrog@users.noreply.github.com>

LibreNMS/Util/StringHelpers.php

@@ -64,6 +64,7 @@ class StringHelpers
             'ntp-client' => 'NTP Client',
             'ntp-server' => 'NTP Server',
             'opengridscheduler' => 'Open Grid Scheduler',
+            'opensearch' => 'Elasticsearch\Opensearch',
             'os-updates' => 'OS Updates',
             'php-fpm' => 'PHP-FPM',
             'pi-hole' => 'Pi-hole',

doc/Extensions/Applications.md

@@ -1214,6 +1214,45 @@ sees them as being.
 For questions about what the various values are/mean, please see the
 nvidia-smi man file under the section covering dmon.
 
+## Opensearch\Elasticsearch
+
+### SNMP Extend
+
+1. Download the script onto the desired host.
+```
+wget https://raw.githubusercontent.com/librenms/librenms-agent/master/snmp/opensearch -O /etc/snmp/opensearch
+```
+
+2. Make it executable
+```
+chmod +x /etc/snmp/opensearch
+```
+
+3. Install the required Perl dependencies.
+```
+# FreeBSD
+pkg install p5-JSON p5-libwww
+# Debian/Ubuntu
+apt-get install libjson-perl libwww-perl
+# cpanm
+cpanm JSON Libwww
+```
+
+4. Update your snmpd.conf.
+```
+extend opensearch /bin/cat /var/cache/opensearch.json
+```
+
+5. Update root crontab with. This is required as it will this will
+likely time out otherwise. Use `*/1` if you want to have the most
+recent stats when polled or to `*/5` if you just want at exactly a 5
+minute interval.
+```
+*/5 * * * * /etc/snmp/opensearch > /var/cache/opensearch.json
+```
+
+6. Enable it or wait for the device to be re-disocvered.
+
 ## Open Grid Scheduler
 
 Shell script to track the OGS/GE jobs running on clusters.

includes/html/graphs/application/opensearch_c_act_pri_shards.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'ActPri Shards',
+        'ds'       => 'c_act_pri_shards',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_act_shards.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Act Shards',
+        'ds'       => 'c_act_pri_shards',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_act_shards_perc.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards %';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Act Shards %',
+        'ds'       => 'c_act_shards_perc',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_all_shards.inc.php

@@ -0,0 +1,49 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Active Primary',
+        'ds'       => 'c_act_pri_shards',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Active',
+        'ds'       => 'c_act_shards',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Relocating',
+        'ds'       => 'c_rel_shards',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Initializing',
+        'ds'       => 'c_init_shards',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Delayed Unass.',
+        'ds'       => 'c_delayed_shards',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Unassigned',
+        'ds'       => 'c_unass_shards',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_data_nodes.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Nodes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Data Nodes',
+        'ds'       => 'c_data_nodes',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_delayed_shards.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Delayed Shards',
+        'ds'       => 'c_delayed_shards',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_in_fl_fetch.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Fetches';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'In Fl. Fetches',
+        'ds'       => 'c_in_fl_fetch',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_init_shards.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Init Shards',
+        'ds'       => 'c_init_shards',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_nodes.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Nodes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Nodes',
+        'ds'       => 'c_nodes',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_pending_tasks.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Pending Tasks',
+        'ds'       => 'c_pending_tasks',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_rel_shards.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Rel Shards',
+        'ds'       => 'c_rel_shards',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_task_max_in_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Milliseconds';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Task Max In Time',
+        'ds'       => 'c_task_max_in_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_c_unass_shards.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Shards';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Unassigned',
+        'ds'       => 'c_unass_shards',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_mem_all.inc.php

@@ -0,0 +1,84 @@
+<?php
+
+$name = 'opensearch';
+$app_id = $app->app_id;
+$unit_text = 'Bytes';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 1;
+$addarea = 0;
+$transparency = 15;
+
+$tqc_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tqc']);
+$trc_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'trc']);
+$tfd_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tfd']);
+$tseg_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+$rrd_list[] = [
+    'filename' => $tqc_rrd_filename,
+    'descr'    => 'Query Cache',
+    'ds'       => 'tqc_size',
+];
+$rrd_list[] = [
+    'filename' => $trc_rrd_filename,
+    'descr'    => 'Request Cache',
+    'ds'       => 'trc_size',
+];
+$rrd_list[] = [
+    'filename' => $tfd_rrd_filename,
+    'descr'    => 'Fielddata',
+    'ds'       => 'tfd_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Segments',
+    'ds'       => 'tseg_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg. Terms',
+    'ds'       => 'tseg_terms_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg. Fields',
+    'ds'       => 'tseg_fields_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg.Term.Vectors',
+    'ds'       => 'tseg_tvector_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg. Norms',
+    'ds'       => 'tseg_norms_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg. Points',
+    'ds'       => 'tseg_points_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg. Doc Vals',
+    'ds'       => 'tseg_docval_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg.Ind.Wrt.',
+    'ds'       => 'tseg_indwrt_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg. Ver. Map.',
+    'ds'       => 'tseg_vermap_size',
+];
+$rrd_list[] = [
+    'filename' => $tseg_rrd_filename,
+    'descr'    => 'Seg. Fixed Bit Set',
+    'ds'       => 'tseg_fbs_size',
+];
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_status.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Status';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'c']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Status',
+        'ds'       => 'status',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tf_periodic.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tf']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Periodic Flush',
+        'ds'       => 'tf_periodic',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tf_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tf']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Flush Time',
+        'ds'       => 'tf_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tf_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tf']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Flush',
+        'ds'       => 'tf_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tfd_evictions.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tfd']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'FD Evictions',
+        'ds'       => 'tfd_evictions',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tfd_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tfd']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Fielddata',
+        'ds'       => 'tfd_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tg_exists_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Get Exists Time',
+        'ds'       => 'tg_exists_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tg_exists_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Get Exists',
+        'ds'       => 'tg_exists_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tg_missing_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Get Missing Time',
+        'ds'       => 'tg_missing_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tg_missing_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Get Missing',
+        'ds'       => 'tg_missing_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tg_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Get Time',
+        'ds'       => 'tg_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tg_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Gets',
+        'ds'       => 'tg_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ti_del_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Delete Time',
+        'ds'       => 'ti_del_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ti_del_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Index Deletes',
+        'ds'       => 'ti_del_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ti_failed.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Failed Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Index Failed',
+        'ds'       => 'ti_failed',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ti_noop_up_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'NoOPs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'NoOP Updates',
+        'ds'       => 'ti_noop_up_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ti_throttled.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Throttled';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Throttled',
+        'ds'       => 'ti_throttled',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ti_throttled_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisec/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Throttled Time',
+        'ds'       => 'ti_throttled_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ti_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Index Time',
+        'ds'       => 'ti_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ti_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Index',
+        'ds'       => 'ti_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_time_all.inc.php

@@ -0,0 +1,90 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 1;
+$addarea = 0;
+$transparency = 15;
+
+$ti_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ti']);
+$ts_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+$tr_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tr']);
+$tf_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tf']);
+$tg_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tg']);
+$tm_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tm']);
+$tw_rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tw']);
+
+$rrd_list[] = [
+    'filename' => $ti_rrd_filename,
+    'descr'    => 'Index',
+    'ds'       => 'ti_time',
+];
+$rrd_list[] = [
+    'filename' => $ti_rrd_filename,
+    'descr'    => 'Delete',
+    'ds'       => 'ti_del_time',
+];
+$rrd_list[] = [
+    'filename' => $ts_rrd_filename,
+    'descr'    => 'Query',
+    'ds'       => 'ts_q_time',
+];
+$rrd_list[] = [
+    'filename' => $ts_rrd_filename,
+    'descr'    => 'Fetch',
+    'ds'       => 'ts_f_time',
+];
+$rrd_list[] = [
+    'filename' => $ts_rrd_filename,
+    'descr'    => 'Scroll',
+    'ds'       => 'ts_sc_time',
+];
+$rrd_list[] = [
+    'filename' => $ts_rrd_filename,
+    'descr'    => 'Suggest',
+    'ds'       => 'ts_su_time',
+];
+$rrd_list[] = [
+    'filename' => $tr_rrd_filename,
+    'descr'    => 'Refresh',
+    'ds'       => 'tr_time',
+];
+$rrd_list[] = [
+    'filename' => $tr_rrd_filename,
+    'descr'    => 'Ext. Refresh',
+    'ds'       => 'tr_ext_time',
+];
+$rrd_list[] = [
+    'filename' => $tf_rrd_filename,
+    'descr'    => 'Flush',
+    'ds'       => 'tf_time',
+];
+$rrd_list[] = [
+    'filename' => $tg_rrd_filename,
+    'descr'    => 'Get',
+    'ds'       => 'tg_time',
+];
+$rrd_list[] = [
+    'filename' => $tg_rrd_filename,
+    'descr'    => 'Get Exists',
+    'ds'       => 'tg_exists_time',
+];
+$rrd_list[] = [
+    'filename' => $tg_rrd_filename,
+    'descr'    => 'Get Missing',
+    'ds'       => 'tg_missing_time',
+];
+$rrd_list[] = [
+    'filename' => $tm_rrd_filename,
+    'descr'    => 'Merges',
+    'ds'       => 'tm_time',
+];
+$rrd_list[] = [
+    'filename' => $tw_rrd_filename,
+    'descr'    => 'Warmer',
+    'ds'       => 'tw_time',
+];
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tm_docs.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tm']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Merged Docs',
+        'ds'       => 'tm_docs',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tm_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tm']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Merges Size',
+        'ds'       => 'tm_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tm_throttled_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tm']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'M.Throt.Size',
+        'ds'       => 'tm_throttled_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tm_throttled_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tm']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'M.Throt.Time',
+        'ds'       => 'tm_throttled_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tm_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tm']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Merges Time',
+        'ds'       => 'tm_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tm_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tm']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Merges',
+        'ds'       => 'tm_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tqc_cache_count.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Cache Count';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tqc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'QC Cache Count',
+        'ds'       => 'tqc_cache_count',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tqc_cache_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Size';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tqc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'QC Size',
+        'ds'       => 'tqc_cache_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tqc_evictions.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Evictions';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tqc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'QC Evictions',
+        'ds'       => 'tqc_evictions',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tqc_hit.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Hits';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tqc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'QC Hits',
+        'ds'       => 'tqc_hit',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tqc_miss.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Misses';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tqc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'QC Misses',
+        'ds'       => 'tqc_miss',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tqc_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tqc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'QC Size',
+        'ds'       => 'tqc_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tqc_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Cached Queries';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tqc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Queries',
+        'ds'       => 'tqc_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tr_ext_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tr']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'External Refresh',
+        'ds'       => 'tr_ext_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tr_ext_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tr']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Ext Refreshes',
+        'ds'       => 'tr_ext_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tr_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisec/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tr']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Index Time',
+        'ds'       => 'tr_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tr_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tr']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Refreshes',
+        'ds'       => 'tr_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_trc_evictions.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'trc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'RC Evictions',
+        'ds'       => 'trc_evictions',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_trc_hits.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'trc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'RC Hits',
+        'ds'       => 'trc_hits',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_trc_misses.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'trc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'RC Misses',
+        'ds'       => 'trc_misses',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_trc_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'trc']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Request Cache',
+        'ds'       => 'trc_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ts_f_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Fetch Time',
+        'ds'       => 'ts_f_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ts_f_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Fetches',
+        'ds'       => 'ts_f_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ts_q_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Query Time',
+        'ds'       => 'ts_q_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ts_q_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Queries',
+        'ds'       => 'ts_q_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ts_sc_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisec/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Scroll Time',
+        'ds'       => 'ts_sc_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ts_sc_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Scrolls / Second';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Scrolls',
+        'ds'       => 'ts_sc_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ts_su_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisec/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Suggest Time',
+        'ds'       => 'ts_su_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ts_su_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Suggests / Second';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ts']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Suggests',
+        'ds'       => 'ts_su_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_count.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Segments';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Segments Count',
+        'ds'       => 'tseg_count',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_docval_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg. Doc Val. Size',
+        'ds'       => 'tseg_docval_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_fbs_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg. Fixed Bit Set Size',
+        'ds'       => 'tseg_fbs_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_fields_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg. Fields Size',
+        'ds'       => 'tseg_fields_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_indwrt_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg.Ind.Wrt.Size',
+        'ds'       => 'tseg_indwrt_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_norms_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg. Norms Size',
+        'ds'       => 'tseg_norms_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_points_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg. Points Size',
+        'ds'       => 'tseg_points_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Segment Size',
+        'ds'       => 'tseg_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_terms_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg. Terms Size',
+        'ds'       => 'tseg_terms_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_tvector_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg.Term.Vect.Size',
+        'ds'       => 'tseg_tvector_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tseg_vermap_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tseg']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Seg.Ver.Map Size',
+        'ds'       => 'tseg_vermap_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tst_res_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tst']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Store Reserved',
+        'ds'       => 'tst_res_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tst_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tst']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Store',
+        'ds'       => 'tst_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ttl_last_mod_age.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Seconds';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ttl']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Last Mod Age',
+        'ds'       => 'ttl_last_mod_age',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ttl_ops.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ttl']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Translog Ops',
+        'ds'       => 'ttl_ops',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ttl_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ttl']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Translog Size',
+        'ds'       => 'ttl_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ttl_uncom_ops.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ttl']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Uncom Ops',
+        'ds'       => 'ttl_uncom_ops',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_ttl_uncom_size.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Bytes';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'ttl']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Uncom Size',
+        'ds'       => 'ttl_uncom_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tw_time.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Millisecs/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tw']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Warmer Time',
+        'ds'       => 'tw_time',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/opensearch_tw_total.inc.php

@@ -0,0 +1,24 @@
+<?php
+
+$name = 'opensearch';
+$unit_text = 'Ops/Sec';
+$colours = 'greens';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'tw']);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Warmer',
+        'ds'       => 'tw_total',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/pages/apps.inc.php

@@ -126,6 +126,17 @@ $graphs['nfs-server'] = [
 $graphs['os-updates'] = [
     'packages',
 ];
+$graphs['opensearch'] = [
+    'status',
+    'c_nodes',
+    'c_data_nodes',
+    'c_pending_tasks',
+    'c_in_fl_fetch',
+    'c_task_max_in_time',
+    'c_act_shards_perc',
+    'mem_all',
+    'time_all',
+];
 $graphs['dhcp-stats'] = [
     'stats',
     'pools_percent',

includes/html/pages/device/apps/opensearch.inc.php

@@ -0,0 +1,251 @@
+<?php
+
+$link_array = [
+    'page'   => 'device',
+    'device' => $device['device_id'],
+    'tab'    => 'apps',
+    'app'    => 'opensearch',
+];
+
+print_optionbar_start();
+echo '<b>Cluster Name:</b> ' . $app->data['cluster'] . '<br>';
+echo '<b>Graph Sets:</b> ';
+echo generate_link('Cluster, ', $link_array);
+$link_array['set'] = 'translog';
+echo generate_link('Translog, ', $link_array);
+$link_array['set'] = 'indexing';
+echo generate_link('Indexing, ', $link_array);
+$link_array['set'] = 'search';
+echo generate_link('Search, ', $link_array);
+$link_array['set'] = 'refresh';
+echo generate_link('Refresh, ', $link_array);
+$link_array['set'] = 'flush';
+echo generate_link('Flush, ', $link_array);
+$link_array['set'] = 'qc';
+echo generate_link('Query_Cache, ', $link_array);
+$link_array['set'] = 'get';
+echo generate_link('Get, ', $link_array);
+$link_array['set'] = 'merges';
+echo generate_link('Merges, ', $link_array);
+$link_array['set'] = 'warmer';
+echo generate_link('Warmer, ', $link_array);
+$link_array['set'] = 'fielddata';
+echo generate_link('Fielddata, ', $link_array);
+$link_array['set'] = 'segments';
+echo generate_link('Segments, ', $link_array);
+$link_array['set'] = 'rc';
+echo generate_link('Request_Cache, ', $link_array);
+$link_array['set'] = 'store';
+echo generate_link('Store, ', $link_array);
+$link_array['set'] = 'time';
+echo generate_link('Time, ', $link_array);
+$link_array['set'] = 'memory';
+echo generate_link('Memory', $link_array);
+print_optionbar_end();
+
+if (isset($vars['set'])) {
+    $graph_set = $vars['set'];
+} else {
+    $graph_set = $vars['cluster'];
+}
+
+if ($graph_set == 'cluster') {
+    $graphs = [
+        'opensearch_c_nodes'=>'Nodes',
+        'opensearch_c_data_nodes'=>'Data Nodes',
+        'opensearch_c_all_shards'=>'Combined Shard Stats',
+        'opensearch_c_pending_tasks'=>'Pending Tasks',
+        'opensearch_c_in_fl_fetch'=>'In Flight Fetches',
+        'opensearch_status'=>'Status: 0=Green, 1=Yellow, 2=Red, 3=Unknown',
+        'opensearch_c_task_max_in_time'=>'Tasks Max Time In Milliseconds',
+        'opensearch_c_act_shards_perc'=>'Active Shards Percentage',
+        'opensearch_c_act_pri_shards'=>'Active Primary Shards',
+        'opensearch_c_act_shards'=>'Active Shards',
+        'opensearch_c_rel_shards'=>'Relocating Shards',
+        'opensearch_c_init_shards'=>'Initializing Shards',
+        'opensearch_c_delayed_shards'=>'Delayed Shards',
+        'opensearch_c_unass_shards'=>'Unassigned Shards',
+    ];
+} elseif ($graph_set == 'translog') {
+    $graphs = [
+        'opensearch_ttl_ops' => 'Translog Operations',
+        'opensearch_ttl_size' => 'Translog Size In Bytes',
+        'opensearch_ttl_uncom_ops' => 'Translog Uncommitted Operations',
+        'opensearch_ttl_uncom_size' => 'Translog Uncommitted Size In Bytes',
+        'opensearch_ttl_last_mod_age' => 'Translog Earliest Last Modified Age',
+    ];
+} elseif ($graph_set == 'indexing') {
+    $graphs = [
+        'opensearch_ti_total' => 'Index',
+        'opensearch_ti_time' => 'Index Time',
+        'opensearch_ti_failed' => 'Index Failed',
+        'opensearch_ti_del_total' => 'Delete',
+        'opensearch_ti_del_time' => 'Delete Time',
+        'opensearch_ti_noop_up_total' => 'NoOP Update',
+        'opensearch_ti_throttled_time' => 'Throttle Time',
+        'opensearch_ti_throttled' => 'Throttled',
+    ];
+} elseif ($graph_set == 'search') {
+    $graphs = [
+        'opensearch_ts_q_total' => 'Queries',
+        'opensearch_ts_q_time' => 'Query Time',
+        'opensearch_ts_f_total' => 'Fetch',
+        'opensearch_ts_f_time' => 'Fetch Time',
+        'opensearch_ts_sc_total' => 'Scrolls',
+        'opensearch_ts_sc_time' => 'Scroll Time',
+        'opensearch_ts_su_total' => 'Suggests',
+        'opensearch_ts_su_time' => 'Suggest Time',
+    ];
+} elseif ($graph_set == 'refresh') {
+    $graphs = [
+        'opensearch_tr_total' => 'Refreshes',
+        'opensearch_tr_time' => 'Refresh Time',
+        'opensearch_tr_ext_total' => 'External',
+        'opensearch_tr_ext_time' => 'External Time',
+    ];
+} elseif ($graph_set == 'flush') {
+    $graphs = [
+        'opensearch_tf_total' => 'Flushes',
+        'opensearch_tf_periodic' => 'Periodic',
+        'opensearch_tf_time' => 'Flush Time',
+    ];
+} elseif ($graph_set == 'qc') {
+    $graphs = [
+        'opensearch_tqc_size' => 'Query Cache Memory Size',
+        'opensearch_tqc_total' => 'Query Cache',
+        'opensearch_tqc_hit' => 'Query Cache Hits',
+        'opensearch_tqc_miss' => 'Query Cache Misses',
+        'opensearch_tqc_cache_size' => 'Query Cache Size',
+        'opensearch_tqc_cache_count' => 'Query Cache Count',
+        'opensearch_tqc_evictions' => 'Query Cache Evictions',
+    ];
+} elseif ($graph_set == 'get') {
+    $graphs = [
+        'opensearch_tg_total' => 'Gets',
+        'opensearch_tg_time' => 'Get Time',
+        'opensearch_tg_exists_total' => 'Exists',
+        'opensearch_tg_exists_time' => 'Get Exists Time',
+        'opensearch_tg_missing_total' => 'Get Missing Total',
+        'opensearch_tg_missing_time' => 'Get Missing Time',
+    ];
+} elseif ($graph_set == 'merges') {
+    $graphs = [
+        'opensearch_tm_total' => 'Merges',
+        'opensearch_tm_time' => 'Merges Time',
+        'opensearch_tm_docs' => 'Merges Docs',
+        'opensearch_tm_size' => 'Merges Size',
+        'opensearch_tm_throttled_time' => 'Merges Throttled Time',
+        'opensearch_tm_throttled_size' => 'Merges Throttled Size',
+    ];
+} elseif ($graph_set == 'warmer') {
+    $graphs = [
+        'opensearch_tw_total' => 'Warmer',
+        'opensearch_tw_time' => 'Warmer Time',
+    ];
+} elseif ($graph_set == 'fielddata') {
+    $graphs = [
+        'opensearch_tfd_size' => 'Fielddata Memory Size',
+        'opensearch_tfd_evictions' => ' Fielddata Evictions',
+    ];
+} elseif ($graph_set == 'segments') {
+    $graphs = [
+        'opensearch_tseg_count' => 'Segments Count',
+        'opensearch_tseg_size' => 'Segments Memory Size',
+        'opensearch_tseg_terms_size' => 'Segments Terms Memory Size',
+        'opensearch_tseg_fields_size' => 'Segments Stored Fields Memory Size',
+        'opensearch_tseg_tvector_size' => 'Segments Term Vertors Memory Size',
+        'opensearch_tseg_norms_size' => 'Segments Norms Memory Size',
+        'opensearch_tseg_points_size' => 'Segments Points Memory Size',
+        'opensearch_tseg_docval_size' => 'Segments Doc Values Memory Size',
+        'opensearch_tseg_indwrt_size' => 'Segments Index Writer Memory Size',
+        'opensearch_tseg_vermap_size' => 'Segments Version Map Memory Size',
+        'opensearch_tseg_fbs_size' => 'Segment Fixed Bit Set Memory Size',
+    ];
+} elseif ($graph_set == 'rc') {
+    $graphs = [
+        'opensearch_trc_size' => 'Request Cache Memory Size',
+        'opensearch_trc_evictions' => 'Request Cache Evictions',
+        'opensearch_trc_hits' => 'Request Cache Hits',
+        'opensearch_trc_misses' => 'Request Cache Misses',
+    ];
+} elseif ($graph_set == 'store') {
+    $graphs = [
+        'opensearch_tst_size' => 'Store Size',
+        'opensearch_tst_res_size' => 'Store Reserved Size',
+    ];
+} elseif ($graph_set == 'time') {
+    $graphs = [
+        'opensearch_c_task_max_in_time'=>'Tasks Max Time',
+        'opensearch_time_all' => 'Combined',
+        'opensearch_ti_time' => 'Index Time',
+        'opensearch_ti_del_time' => 'Delete Time',
+        'opensearch_ti_throttled_time' => 'Throttle Time',
+        'opensearch_ts_q_time' => 'Query Time',
+        'opensearch_ts_f_time' => 'Fetch Time',
+        'opensearch_ts_sc_time' => 'Scroll Time',
+        'opensearch_ts_su_time' => 'Suggest Time',
+        'opensearch_tr_time' => 'Refresh Time',
+        'opensearch_tr_ext_time' => 'External Time',
+        'opensearch_tf_time' => 'Flush Time',
+        'opensearch_tg_time' => 'Get Time',
+        'opensearch_tg_exists_time' => 'Get Exists Time',
+        'opensearch_tg_missing_time' => 'Get Missing Time',
+        'opensearch_tm_time' => 'Merges Time',
+        'opensearch_tm_throttled_time' => 'Merges Throttled Time',
+        'opensearch_tw_time' => 'Warmer Time',
+    ];
+} elseif ($graph_set == 'memory') {
+    $graphs = [
+        'opensearch_mem_all' => 'Combined',
+        'opensearch_tqc_size' => 'Query Cache Size',
+        'opensearch_trc_size' => 'Request Cache Memory Size',
+        'opensearch_tfd_size' => 'Field Data Memory Size',
+        'opensearch_tseg_size' => 'Segments Memory Size',
+        'opensearch_tseg_terms_size' => 'Segments Terms Memory Size',
+        'opensearch_tseg_fields_size' => 'Segments Stored Fields Memory Size',
+        'opensearch_tseg_tvector_size' => 'Segments Term Vertors Memory Size',
+        'opensearch_tseg_norms_size' => 'Segments Norms Memory Size',
+        'opensearch_tseg_points_size' => 'Segments Points Memory Size',
+        'opensearch_tseg_docval_size' => 'Segments Doc Values Memory Size',
+        'opensearch_tseg_indwrt_size' => 'Segments Index Writer Memory Size',
+        'opensearch_tseg_vermap_size' => 'Segments Version Map Memory Size',
+        'opensearch_tseg_fbs_size' => 'Segment Fixed Bit Set Memory Size',
+    ];
+} else {
+    $graphs = [
+        'opensearch_c_nodes'=>'Nodes',
+        'opensearch_c_data_nodes'=>'Data Nodes',
+        'opensearch_c_all_shards'=>'Combined Shard Stats',
+        'opensearch_c_pending_tasks'=>'Pending Tasks',
+        'opensearch_c_in_fl_fetch'=>'In Flight Fetches',
+        'opensearch_status'=>'Status: 0=Green, 1=Yellow, 2=Red, 3=Unknown',
+        'opensearch_c_task_max_in_time'=>'Tasks Max Time In Milliseconds',
+        'opensearch_c_act_shards_perc'=>'Active Shards Percentage',
+        'opensearch_c_act_pri_shards'=>'Active Primary Shards',
+        'opensearch_c_act_shards'=>'Active Shards',
+        'opensearch_c_rel_shards'=>'Relocating Shards',
+        'opensearch_c_init_shards'=>'Initializing Shards',
+        'opensearch_c_delayed_shards'=>'Delayed Shards',
+        'opensearch_c_unass_shards'=>'Unassigned Shards',
+    ];
+}
+
+foreach ($graphs as $key => $text) {
+    $graph_type = $key;
+    $graph_array['height'] = '100';
+    $graph_array['width'] = '215';
+    $graph_array['to'] = \LibreNMS\Config::get('time.now');
+    $graph_array['id'] = $app->app_id;
+    $graph_array['type'] = 'application_' . $key;
+
+    echo '<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">' . $text . '</h3>
+    </div>
+    <div class="panel-body">
+    <div class="row">';
+    include 'includes/html/print-graphrow.inc.php';
+    echo '</div>';
+    echo '</div>';
+    echo '</div>';
+}

includes/polling/applications/opensearch.inc.php

@@ -0,0 +1,295 @@
+<?php
+
+use LibreNMS\Exceptions\JsonAppException;
+use LibreNMS\RRD\RrdDefinition;
+
+$name = 'opensearch';
+
+try {
+    $returned = json_app_get($device, 'opensearch');
+} catch (JsonAppException $e) {
+    echo PHP_EOL . $name . ':' . $e->getCode() . ':' . $e->getMessage() . PHP_EOL;
+    update_application($app, $e->getCode() . ':' . $e->getMessage(), []); // Set empty metrics and error message
+
+    return;
+}
+
+$data = $returned['data'];
+
+$rrd_name = ['app', $name, $app->app_id, 'c'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('c_nodes', 'GAUGE', 0)
+    ->addDataset('c_data_nodes', 'GAUGE', 0)
+    ->addDataset('c_act_pri_shards', 'GAUGE', 0)
+    ->addDataset('c_act_shards', 'GAUGE', 0)
+    ->addDataset('c_rel_shards', 'GAUGE', 0)
+    ->addDataset('c_init_shards', 'GAUGE', 0)
+    ->addDataset('c_delayed_shards', 'GAUGE', 0)
+    ->addDataset('c_unass_shards', 'GAUGE', 0)
+    ->addDataset('c_pending_tasks', 'GAUGE', 0)
+    ->addDataset('c_in_fl_fetch', 'GAUGE', 0)
+    ->addDataset('c_task_max_in_time', 'GAUGE', 0)
+    ->addDataset('c_act_shards_perc', 'GAUGE', 0)
+    ->addDataset('status', 'GAUGE', 0);
+$metrics = [
+    'c_nodes' => $data['c_nodes'],
+    'c_data_nodes' => $data['c_data_nodes'],
+    'c_act_pri_shards' => $data['c_act_pri_shards'],
+    'c_act_shards' => $data['c_act_shards'],
+    'c_rel_shards' => $data['c_rel_shards'],
+    'c_init_shards' => $data['c_init_shards'],
+    'c_delayed_shards' => $data['c_delayed_shards'],
+    'c_unass_shards' => $data['c_unass_shards'],
+    'c_pending_tasks' => $data['c_pending_tasks'],
+    'c_in_fl_fetch' => $data['c_in_fl_fetch'],
+    'c_task_max_in_time' => $data['c_task_max_in_time'],
+    'c_act_shards_perc' => $data['c_act_shards_perc'],
+    'status' => $data['status'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'ttl'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('ttl_ops', 'DERIVE', 0)
+    ->addDataset('ttl_size', 'GAUGE', 0)
+    ->addDataset('ttl_uncom_ops', 'GAUGE', 0)
+    ->addDataset('ttl_uncom_size', 'GAUGE', 0)
+    ->addDataset('ttl_last_mod_age', 'GAUGE', 0);
+$metrics = [
+    'ttl_ops' => $data['ttl_ops'],
+    'ttl_size' => $data['ttl_size'],
+    'ttl_uncom_ops' => $data['ttl_uncom_ops'],
+    'ttl_uncom_size' => $data['ttl_uncom_size'],
+    'ttl_last_mod_age' => $data['ttl_last_mod_age'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'ti'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('ti_total', 'DERIVE', 0)
+    ->addDataset('ti_time', 'DERIVE', 0)
+    ->addDataset('ti_failed', 'DERIVE', 0)
+    ->addDataset('ti_del_total', 'DERIVE', 0)
+    ->addDataset('ti_del_time', 'DERIVE', 0)
+    ->addDataset('ti_noop_up_total', 'DERIVE', 0)
+    ->addDataset('ti_throttled_time', 'DERIVE', 0)
+    ->addDataset('ti_throttled', 'GAUGE', 0);
+$metrics = [
+    'ti_total' => $data['ti_total'],
+    'ti_time' => $data['ti_time'],
+    'ti_failed' => $data['ti_failed'],
+    'ti_del_total' => $data['ti_del_total'],
+    'ti_del_time' => $data['ti_del_time'],
+    'ti_noop_up_total' => $data['ti_noop_up_total'],
+    'ti_throttled_time' => $data['ti_throttled_time'],
+    'ti_throttled' => $data['ti_throttled'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'ts'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('ts_q_total', 'DERIVE', 0)
+    ->addDataset('ts_q_time', 'DERIVE', 0)
+    ->addDataset('ts_f_total', 'DERIVE', 0)
+    ->addDataset('ts_f_time', 'DERIVE', 0)
+    ->addDataset('ts_sc_total', 'DERIVE', 0)
+    ->addDataset('ts_sc_time', 'DERIVE', 0)
+    ->addDataset('ts_su_total', 'DERIVE', 0)
+    ->addDataset('ts_su_time', 'DERIVE', 0);
+$metrics = [
+    'ts_q_total' => $data['ts_q_total'],
+    'ts_q_time' => $data['ts_q_time'],
+    'ts_f_total' => $data['ts_f_total'],
+    'ts_f_time' => $data['ts_f_time'],
+    'ts_sc_total' => $data['ts_sc_total'],
+    'ts_sc_time' => $data['ts_sc_time'],
+    'ts_su_total' => $data['ts_su_total'],
+    'ts_su_time' => $data['ts_su_time'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tr'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tr_total', 'DERIVE', 0)
+    ->addDataset('tr_time', 'DERIVE', 0)
+    ->addDataset('tr_ext_total', 'DERIVE', 0)
+    ->addDataset('tr_ext_time', 'DERIVE', 0);
+$metrics = [
+    'tr_total' => $data['tr_total'],
+    'tr_time' => $data['tr_time'],
+    'tr_ext_total' => $data['tr_ext_total'],
+    'tr_ext_time' => $data['tr_ext_time'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tf'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tf_total', 'DERIVE', 0)
+    ->addDataset('tf_periodic', 'DERIVE', 0)
+    ->addDataset('tf_time', 'DERIVE', 0);
+$metrics = [
+    'tf_total' => $data['tf_total'],
+    'tf_periodic' => $data['tf_periodic'],
+    'tf_time' => $data['tf_time'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tqc'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tqc_size', 'GAUGE', 0)
+    ->addDataset('tqc_total', 'GAUGE', 0)
+    ->addDataset('tqc_hit', 'DERIVE', 0)
+    ->addDataset('tqc_miss', 'DERIVE', 0)
+    ->addDataset('tqc_cache_size', 'GAUGE', 0)
+    ->addDataset('tqc_cache_count', 'GAUGE', 0)
+    ->addDataset('tqc_evictions', 'DERIVE', 0);
+$metrics = [
+    'tqc_size' => $data['tqc_size'],
+    'tqc_total' => $data['tqc_total'],
+    'tqc_hit' => $data['tqc_hit'],
+    'tqc_miss' => $data['tqc_miss'],
+    'tqc_cache_size' => $data['tqc_cache_size'],
+    'tqc_cache_count' => $data['tqc_cache_count'],
+    'tqc_evictions' => $data['tqc_evictions'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tg'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tg_total', 'DERIVE', 0)
+    ->addDataset('tg_time', 'DERIVE', 0)
+    ->addDataset('tg_exists_total', 'DERIVE', 0)
+    ->addDataset('tg_exists_time', 'DERIVE', 0)
+    ->addDataset('tg_missing_total', 'DERIVE', 0)
+    ->addDataset('tg_missing_time', 'DERIVE', 0);
+$metrics = [
+    'tg_total' => $data['tg_total'],
+    'tg_time' => $data['tg_time'],
+    'tg_exists_total' => $data['tg_exists_total'],
+    'tg_exists_time' => $data['tg_exists_time'],
+    'tg_missing_total' => $data['tg_missing_total'],
+    'tg_missing_time' => $data['tg_missing_time'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tm'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tm_total', 'DERIVE', 0)
+    ->addDataset('tm_time', 'DERIVE', 0)
+    ->addDataset('tm_docs', 'DERIVE', 0)
+    ->addDataset('tm_size', 'DERIVE', 0)
+    ->addDataset('tm_throttled_time', 'DERIVE', 0)
+    ->addDataset('tm_throttled_size', 'DERIVE', 0);
+$metrics = [
+    'tm_total' => $data['tm_total'],
+    'tm_time' => $data['tm_time'],
+    'tm_docs' => $data['tm_docs'],
+    'tm_size' => $data['tm_size'],
+    'tm_throttled_time' => $data['tm_throttled_time'],
+    'tm_throttled_size' => $data['tm_throttled_size'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tw'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tw_total', 'DERIVE', 0)
+    ->addDataset('tw_time', 'DERIVE', 0);
+$metrics = [
+    'tw_total' => $data['tw_total'],
+    'tw_time' => $data['tw_time'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tfd'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tfd_size', 'GAUGE', 0)
+    ->addDataset('tfd_evictions', 'DERIVE', 0);
+$metrics = [
+    'tfd_size' => $data['tfd_size'],
+    'tfd_evictions' => $data['tfd_evictions'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tseg'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tseg_count', 'GAUGE', 0)
+    ->addDataset('tseg_size', 'GAUGE', 0)
+    ->addDataset('tseg_terms_size', 'GAUGE', 0)
+    ->addDataset('tseg_fields_size', 'GAUGE', 0)
+    ->addDataset('tseg_tvector_size', 'GAUGE', 0)
+    ->addDataset('tseg_norms_size', 'GAUGE', 0)
+    ->addDataset('tseg_points_size', 'GAUGE', 0)
+    ->addDataset('tseg_docval_size', 'GAUGE', 0)
+    ->addDataset('tseg_indwrt_size', 'GAUGE', 0)
+    ->addDataset('tseg_vermap_size', 'GAUGE', 0)
+    ->addDataset('tseg_fbs_size', 'GAUGE', 0);
+$metrics = [
+    'tseg_count' => $data['tseg_count'],
+    'tseg_size' => $data['tseg_size'],
+    'tseg_terms_size' => $data['tseg_terms_size'],
+    'tseg_fields_size' => $data['tseg_fields_size'],
+    'tseg_tvector_size' => $data['tseg_tvector_size'],
+    'tseg_norms_size' => $data['tseg_norms_size'],
+    'tseg_points_size' => $data['tseg_points_size'],
+    'tseg_docval_size' => $data['tseg_docval_size'],
+    'tseg_indwrt_size' => $data['tseg_indwrt_size'],
+    'tseg_vermap_size' => $data['tseg_vermap_size'],
+    'tseg_fbs_size' => $data['tseg_fbs_size'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'trc'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('trc_size', 'GAUGE', 0)
+    ->addDataset('trc_evictions', 'DERIVE', 0)
+    ->addDataset('trc_hits', 'DERIVE', 0)
+    ->addDataset('trc_misses', 'DERIVE', 0);
+$metrics = [
+    'trc_size' => $data['trc_size'],
+    'trc_evictions' => $data['trc_evictions'],
+    'trc_hits' => $data['trc_hits'],
+    'trc_misses' => $data['trc_misses'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+$rrd_name = ['app', $name, $app->app_id, 'tst'];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('tst_size', 'GAUGE', 0)
+    ->addDataset('tst_res_size', 'GAUGE', 0);
+$metrics = [
+    'tst_size' => $data['tst_size'],
+    'tst_res_size' => $data['tst_res_size'],
+];
+$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $metrics);
+
+// save clustername upon changes and log it post initial set
+if (isset($app->data['cluster'])) {
+    if ($app->data['cluster'] != $returned['data']['cluster_name']) {
+        log_event('Elastic/Opensearch: Cluster name changed to "' . $returned['data']['cluster_name'] . '"', $device, 'application');
+
+        // save the found cluster name
+        $app->data = ['cluster' => $returned['data']['cluster_name']];
+    }
+} else {
+    $app->data = ['cluster' => $returned['data']['cluster_name']];
+}
+
+//
+// update the app metrics
+//
+unset($returned['data']['cluster_name']);
+update_application($app, 'OK', $returned['data']);

misc/alert_rules.json

@@ -533,6 +533,16 @@
     "name": "MySQL Server not responding",
     "severity":"critical"
   },
+  {
+    "rule": "applications.app_type = \"opensearch\" && application_metrics.metric = \".status\" && application_metrics.value = \"1\"",
+    "name": "Opensearch/Elasticsearch Cluster Status Yellow",
+    "severity": "warning"
+  },
+  {
+    "rule": "applications.app_type = \"opensearch\" && application_metrics.metric = \".status\" && application_metrics.value = \"2\"",
+    "name": "Opensearch/Elasticsearch Cluster Status Red",
+    "severity": "critical"
+  },
   {
     "rule": "applications.app_type = \"sagan\" && application_metrics.metric = \".total_alert\" && application_metrics.value = \"1\"",
     "name": "Sagan has a WARNING alert",

tests/data/linux_opensearch-v1.json

@@ -0,0 +1,512 @@
+{
+    "applications": {
+        "discovery": {
+            "applications": [
+                {
+                    "app_type": "opensearch",
+                    "app_state": "UNKNOWN",
+                    "discovered": 1,
+                    "app_state_prev": null,
+                    "app_status": "",
+                    "app_instance": "",
+                    "data": null
+                }
+            ]
+        },
+        "poller": {
+            "applications": [
+                {
+                    "app_type": "opensearch",
+                    "app_state": "OK",
+                    "discovered": 1,
+                    "app_state_prev": "UNKNOWN",
+                    "app_status": "",
+                    "app_instance": "",
+                    "data": "{\"cluster\":\"vulpes\"}"
+                }
+            ],
+            "application_metrics": [
+                {
+                    "metric": "c_act_pri_shards",
+                    "value": 262,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_act_shards",
+                    "value": 262,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_act_shards_perc",
+                    "value": 52.929292929292899,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_data_nodes",
+                    "value": 1,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_delayed_shards",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_in_fl_fetch",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_init_shards",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_nodes",
+                    "value": 1,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_pending_tasks",
+                    "value": 212,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_rel_shards",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "c_task_max_in_time",
+                    "value": 76717,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "status",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tf_periodic",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tf_time",
+                    "value": 79694,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tf_total",
+                    "value": 261,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tfd_evictions",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tfd_size",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tg_exists_time",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tg_exists_total",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tg_missing_time",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tg_missing_total",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tg_time",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tg_total",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ti_del_time",
+                    "value": 100,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ti_del_total",
+                    "value": 285,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ti_failed",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ti_noop_up_total",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ti_throttled",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ti_throttled_time",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ti_time",
+                    "value": 2643104411,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ti_total",
+                    "value": 9212092,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tm_docs",
+                    "value": 434455886,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tm_size",
+                    "value": 136565980034,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tm_throttled_size",
+                    "value": 4850140622,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tm_throttled_time",
+                    "value": 76881,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tm_time",
+                    "value": 498616621,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tm_total",
+                    "value": 180358,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tqc_cache_count",
+                    "value": 4,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tqc_cache_size",
+                    "value": 4,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tqc_evictions",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tqc_hit",
+                    "value": 4,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tqc_miss",
+                    "value": 378,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tqc_size",
+                    "value": 2160,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tqc_total",
+                    "value": 382,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tr_ext_time",
+                    "value": 700610451,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tr_ext_total",
+                    "value": 155976,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tr_time",
+                    "value": 690103910,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tr_total",
+                    "value": 156038,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "trc_evictions",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "trc_hits",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "trc_misses",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "trc_size",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ts_f_time",
+                    "value": 41073,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ts_f_total",
+                    "value": 876,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ts_q_time",
+                    "value": 11234,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ts_q_total",
+                    "value": 876,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ts_sc_time",
+                    "value": 25901,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ts_sc_total",
+                    "value": 520,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ts_su_time",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ts_su_total",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_count",
+                    "value": 2066,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_docval_size",
+                    "value": 11440192,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_fbs_size",
+                    "value": 15304,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_fields_size",
+                    "value": 1069920,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_indwrt_size",
+                    "value": 8396888,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_norms_size",
+                    "value": 12096,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_points_size",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_size",
+                    "value": 87278368,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_terms_size",
+                    "value": 74756160,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_tvector_size",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tseg_vermap_size",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tst_res_size",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tst_size",
+                    "value": 18325569074,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ttl_last_mod_age",
+                    "value": 26686572,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ttl_ops",
+                    "value": 259545,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ttl_size",
+                    "value": 162559260,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ttl_uncom_ops",
+                    "value": 259545,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "ttl_uncom_size",
+                    "value": 162559260,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tw_time",
+                    "value": 111862,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                },
+                {
+                    "metric": "tw_total",
+                    "value": 155742,
+                    "value_prev": null,
+                    "app_type": "opensearch"
+                }
+            ]
+        }
+    }
+}

tests/snmpsim/linux_opensearch-v1.snmprec

@@ -0,0 +1,10 @@
+1.3.6.1.2.1.1.1.0|4|Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64
+1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10
+1.3.6.1.2.1.1.3.0|67|77550514
+1.3.6.1.2.1.1.4.0|4|<private>
+1.3.6.1.2.1.1.5.0|4|<private>
+1.3.6.1.2.1.1.6.0|4|<private>
+1.3.6.1.2.1.25.1.1.0|67|77552962
+1.3.6.1.4.1.8072.1.3.2.2.1.21.6.100.105.115.116.114.111|2|1
+1.3.6.1.4.1.8072.1.3.2.2.1.21.10.111.112.101.110.115.101.97.114.99.104|2|1
+1.3.6.1.4.1.8072.1.3.2.3.1.2.10.111.112.101.110.115.101.97.114.99.104|4x|7b2264617461223a7b22635f6163745f7072695f736861726473223a3236322c22635f6163745f736861726473223a3236322c22635f6163745f7368617264735f70657263223a35322e393239323932393239323932392c22635f646174615f6e6f646573223a312c22635f64656c617965645f736861726473223a302c22635f696e5f666c5f6665746368223a302c22635f696e69745f736861726473223a302c22635f6e6f646573223a312c22635f70656e64696e675f7461736b73223a3231322c22635f72656c5f736861726473223a302c22635f7461736b5f6d61785f696e5f74696d65223a37363731372c22636c75737465725f6e616d65223a2276756c706573222c22737461747573223a302c2274665f706572696f646963223a302c2274665f74696d65223a37393639342c2274665f746f74616c223a3236312c227466645f6576696374696f6e73223a302c227466645f73697a65223a302c2274675f6578697374735f74696d65223a302c2274675f6578697374735f746f74616c223a302c2274675f6d697373696e675f74696d65223a302c2274675f6d697373696e675f746f74616c223a302c2274675f74696d65223a302c2274675f746f74616c223a302c2274695f64656c5f74696d65223a3130302c2274695f64656c5f746f74616c223a3238352c2274695f6661696c6564223a302c2274695f6e6f6f705f75705f746f74616c223a302c2274695f7468726f74746c6564223a302c2274695f7468726f74746c65645f74696d65223a302c2274695f74696d65223a323634333130343431312c2274695f746f74616c223a393231323039322c22746d5f646f6373223a3433343435353838362c22746d5f73697a65223a3133363536353938303033342c22746d5f7468726f74746c65645f73697a65223a343835303134303632322c22746d5f7468726f74746c65645f74696d65223a37363838312c22746d5f74696d65223a3439383631363632312c22746d5f746f74616c223a3138303335382c227471635f63616368655f636f756e74223a342c227471635f63616368655f73697a65223a342c227471635f6576696374696f6e73223a302c227471635f686974223a342c227471635f6d697373223a3337382c227471635f73697a65223a323136302c227471635f746f74616c223a3338322c2274725f6578745f74696d65223a3730303631303435312c2274725f6578745f746f74616c223a3135353937362c2274725f74696d65223a3639303130333931302c2274725f746f74616c223a3135363033382c227472635f6576696374696f6e73223a302c227472635f68697473223a302c227472635f6d6973736573223a302c227472635f73697a65223a302c2274735f665f74696d65223a34313037332c2274735f665f746f74616c223a3837362c2274735f715f74696d65223a31313233342c2274735f715f746f74616c223a3837362c2274735f73635f74696d65223a32353930312c2274735f73635f746f74616c223a3532302c2274735f73755f74696d65223a302c2274735f73755f746f74616c223a302c22747365675f636f756e74223a323036362c22747365675f646f6376616c5f73697a65223a31313434303139322c22747365675f6662735f73697a65223a31353330342c22747365675f6669656c64735f73697a65223a313036393932302c22747365675f696e647772745f73697a65223a383339363838382c22747365675f6e6f726d735f73697a65223a31323039362c22747365675f706f696e74735f73697a65223a302c22747365675f73697a65223a38373237383336382c22747365675f7465726d735f73697a65223a37343735363136302c22747365675f74766563746f725f73697a65223a302c22747365675f7665726d61705f73697a65223a302c227473745f7265735f73697a65223a302c227473745f73697a65223a31383332353536393037342c2274746c5f6c6173745f6d6f645f616765223a32363638363537322c2274746c5f6f7073223a3235393534352c2274746c5f73697a65223a3136323535393236302c2274746c5f756e636f6d5f6f7073223a3235393534352c2274746c5f756e636f6d5f73697a65223a3136323535393236302c2274775f74696d65223a3131313836322c2274775f746f74616c223a3135353734327d2c2264617465223a7b7d2c226572726f72223a302c226572726f72537472696e67223a22222c2276657273696f6e223a317d0a