Refactor Datastores to allow future improvements. OpenTSDB Tags. (#11283)

* Datastores to object oriented code, using the Laravel IoC container

Change instantiation
better DI
move OpenTSDB

Small re-orgs

remove unused stuff

Fix graphs and other scripts

Use DI for all except rrd

fix up connection error handling

Add tests, fix up a "few" things
Add Config::forget()

Style fixes

Don't reference legacy code

remove accidental code paste

Add datastores phpunit groups

some tests

* rebase fixes

* some test fixes

* shorter tests

* shorter tests

* Don't except when rrdtool can't be started.

* restore tests

* fix rrd tests

* fix iterable change upstream

* fix isValidDataset

* fix invalid data bug

* fix mysql incorrect ds

* fix issue with data that is too long

* use regular data_update()

* Use log facade

* OpenTSDB mis-ordered arguments fix

* Making a singleton with different options makes different singletons.  Just use the global config settings to disable datastores.

* only filter tags for datastores that won't it don't modify the tags permanently

* Update copyrights to include original authors.

* Stats for all datastores

* Fix mysql sends different rrd / other ds names

* fix snmp last stats not initialized
remove unused function

* remove unused function and move single use function closer to its use

* InfluxDB does not need to update null or U values.
Skip write if all fields are empty

* Fix smart value checks

* fix style issues

* Make sure port data is stored the same way as before for Graphite and OpenTSDB
Add ifIndex tag to all to be compatible

* Missed rrdtool_tune() call

* Test update WIP

* OpenTSDB now includes tags

* fix style
This commit is contained in:
Tony Murray 2020-03-16 09:17:58 -05:00 committed by GitHub
parent 5deda8646a
commit df16de9d2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 2531 additions and 1008 deletions

View File

@ -0,0 +1,81 @@
<?php
/**
* Measurement.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Measure;
class Measurement
{
private $start;
private $type;
private $duration;
private function __construct(string $type)
{
$this->type = $type;
$this->start = microtime(true);
}
/**
* Start the timer for a new operation
*
* @param string $type
* @return static
*/
public static function start(string $type)
{
return new static($type);
}
/**
* End the timer for this operation
*
* @return $this
*/
public function end()
{
$this->duration = microtime(true) - $this->start;
return $this;
}
/**
* Get the duration of the operation
*
* @return float
*/
public function getDuration()
{
return $this->duration;
}
/**
* Get the type of the operation
*
* @return string
*/
public function getType()
{
return $this->type;
}
}

View File

@ -0,0 +1,75 @@
<?php
/**
* MeasurementCollection.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Measure;
use Illuminate\Support\Collection;
class MeasurementCollection extends Collection
{
public function getTotalCount()
{
return $this->sumStat('getCount');
}
public function getTotalDuration()
{
return $this->sumStat('getDuration');
}
public function getCountDiff()
{
return $this->sumStat('getCountDiff');
}
public function getDurationDiff()
{
return $this->sumStat('getDurationDiff');
}
public function checkpoint()
{
$this->each->checkpoint();
}
public function record(Measurement $measurement)
{
$type = $measurement->getType();
if (!$this->has($type)) {
$this->put($type, new MeasurementSummary($type));
}
$this->get($type)->add($measurement);
}
private function sumStat($function)
{
return $this->reduce(function ($sum, $measurement) use ($function) {
$sum += $measurement->$function();
return $sum;
}, 0);
}
}

View File

@ -0,0 +1,92 @@
<?php
/**
* MeasurementSummary.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Measure;
class MeasurementSummary
{
private $type;
private $count = 0;
private $duration = 0.0;
private $checkpointCount = 0;
private $checkpointDuration = 0.0;
public function __construct(string $type)
{
$this->type = $type;
}
public function add(Measurement $measurement)
{
$this->count++;
$this->duration += $measurement->getDuration();
}
/**
* Get the measurement summary
* ['count' => #, 'duration' => s]
*
* @return array
*/
public function get()
{
return [
'count' => $this->count,
'duration' => $this->duration,
];
}
public function getCount()
{
return $this->count;
}
public function getType()
{
return $this->type;
}
public function getDuration()
{
return $this->duration;
}
public function checkpoint()
{
$this->checkpointCount = $this->count;
$this->checkpointDuration = $this->duration;
}
public function getCountDiff()
{
return $this->count - $this->checkpointCount;
}
public function getDurationDiff()
{
return $this->duration - $this->checkpointDuration;
}
}

View File

@ -0,0 +1,64 @@
<?php
/**
* BaseDatastore.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use LibreNMS\Data\Measure\Measurement;
use LibreNMS\Data\Measure\MeasurementCollection;
use LibreNMS\Interfaces\Data\Datastore as DatastoreContract;
abstract class BaseDatastore implements DatastoreContract
{
private $stats;
public function __construct()
{
$this->stats = new MeasurementCollection();
}
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags()
{
return true;
}
/**
* Record statistics for operation
* @param Measurement $stat
*/
protected function recordStatistic(Measurement $stat)
{
$this->stats->record($stat);
}
public function getStats()
{
return $this->stats;
}
}

View File

@ -0,0 +1,159 @@
<?php
/**
* Datastore.php
*
* Aggregates all enabled datastores and dispatches data to them
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use LibreNMS\Config;
use LibreNMS\Interfaces\Data\Datastore as DatastoreContract;
class Datastore
{
protected $stores;
/**
* Initialize and create the Datastore(s)
*
* @param array $options
* @return DatastoreContract
*/
public static function init($options = [])
{
$opts = [
'r' => 'rrd.enable',
'f' => 'influxdb.enable',
'p' => 'prometheus.enable',
'g' => 'graphite.enable',
];
foreach ($opts as $opt => $setting) {
if (isset($options[$opt])) {
Config::set($setting, false);
}
}
return app('Datastore');
}
public static function terminate()
{
\Rrd::close();
}
/**
* Datastore constructor.
* @param array $datastores Implement DatastoreInterface
*/
public function __construct($datastores)
{
$this->stores = $datastores;
}
/**
* Disable a datastore for the rest of this run
*
* @param string $name
*/
public function disable($name)
{
$store = app("LibreNMS\\Data\\Store\\$name");
$position = array_search($store, $this->stores);
if ($position !== false) {
c_echo("[%g$name Disabled%n]\n");
unset($this->stores[$position]);
} else {
c_echo("[%g$name is not a valid datastore name%n]\n");
}
}
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields)
{
// convenience conversion to allow calling with a single value, so, e.g., these are equivalent:
// data_update($device, 'mymeasurement', $tags, 1234);
// AND
// data_update($device, 'mymeasurement', $tags, array('mymeasurement' => 1234));
if (!is_array($fields)) {
$fields = [$measurement => $fields];
}
foreach ($this->stores as $store) {
/** @var DatastoreContract $store */
// rrdtool_data_update() will only use the tags it deems relevant, so we pass all of them.
// However, influxdb saves all tags, so we filter out the ones beginning with 'rrd_'.
$temp_tags = $store->wantsRrdTags() ? $tags : $this->rrdTagFilter($tags);
$store->put($device, $measurement, $temp_tags, $fields);
}
}
/**
* Filter all elements with keys that start with 'rrd_'
*
* @param array $arr input array
* @return array Copy of $arr with all keys beginning with 'rrd_' removed.
*/
private function rrdTagFilter($arr)
{
$result = [];
foreach ($arr as $k => $v) {
if (strpos($k, 'rrd_') === 0) {
continue;
}
$result[$k] = $v;
}
return $result;
}
/**
* Get all the active data stores
*
* @return array
*/
public function getStores()
{
return $this->stores;
}
public function getStats()
{
return array_reduce($this->stores, function ($result, DatastoreContract $store) {
$result[$store->getName()] = $store->getStats();
return $result;
}, []);
}
}

View File

@ -0,0 +1,146 @@
<?php
/**
* Graphite.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @copyright 2017 Falk Stern <https://github.com/fstern/>
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use Carbon\Carbon;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use Log;
class Graphite extends BaseDatastore
{
protected $connection;
protected $prefix;
public function __construct(\Socket\Raw\Factory $socketFactory)
{
parent::__construct();
$host = Config::get('graphite.host');
$port = Config::get('graphite.port', 2003);
try {
$this->connection = $socketFactory->createClient("$host:$port");
} catch (\Exception $e) {
d_echo($e->getMessage());
}
if ($this->connection) {
Log::notice("Graphite connection made to $host");
} else {
Log::error("Graphite connection to $host has failed!");
}
$this->prefix = Config::get('graphite.prefix', '');
}
public function getName()
{
return 'Graphite';
}
public static function isEnabled()
{
return Config::get('graphite.enable', false);
}
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields)
{
if (!$this->connection) {
d_echo("Graphite Error: not connected\n");
return;
}
$timestamp = Carbon::now()->timestamp;
if ($measurement == 'ports') {
$measurement = 'ports|' . $tags['ifName'];
}
// metrics will be built as prefix.hostname.measurement.field value timestamp
// metric fields can not contain . as this is used by graphite as a field separator
$hostname = preg_replace('/\./', '_', $device['hostname']);
$measurement = preg_replace(['/\./', '/\//'], '_', $measurement);
$measurement = preg_replace('/\|/', '.', $measurement);
$measurement_name = preg_replace('/\./', '_', $tags['rrd_name']);
if (is_array($measurement_name)) {
$ms_name = implode(".", $measurement_name);
} else {
$ms_name = $measurement_name;
}
// remove the port-id tags from the metric
if (preg_match('/^port-id\d+/', $ms_name)) {
$ms_name = "";
}
foreach ($fields as $k => $v) {
// Send zero for fields without values
if (empty($v)) {
$v = 0;
}
$metric = implode(".", array_filter([$this->prefix, $hostname, $measurement, $ms_name, $k]));
$this->writeData($metric, $v, $timestamp);
}
}
/**
* @param $metric
* @param $value
* @param $timestamp
*/
private function writeData($metric, $value, $timestamp)
{
try {
$stat = Measurement::start('write');
// Further sanitize the full metric before sending, whitespace isn't allowed
$metric = preg_replace('/\s+/', '_', $metric);
$line = implode(" ", [$metric, $value, $timestamp]);
Log::debug("Sending to Graphite: $line\n");
$this->connection->write("$line\n");
$this->recordStatistic($stat->end());
} catch (\Socket\Raw\Exception $e) {
Log::error('Graphite write error: ' . $e->getMessage());
}
}
}

View File

@ -0,0 +1,189 @@
<?php
/**
* InfluxDB.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @copyright 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use InfluxDB\Client;
use InfluxDB\Driver\UDP;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use Log;
class InfluxDB extends BaseDatastore
{
/** @var \InfluxDB\Database $connection */
private $connection;
public function __construct(\InfluxDB\Database $influx)
{
parent::__construct();
$this->connection = $influx;
// if the database doesn't exist, create it.
try {
if (!$influx->exists()) {
$influx->create();
}
} catch (\Exception $e) {
Log::warning('InfluxDB: Could not create database');
}
}
public function getName()
{
return 'InfluxDB';
}
public static function isEnabled()
{
return Config::get('influxdb.enable', false);
}
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields)
{
$stat = Measurement::start('write');
$tmp_fields = [];
$tmp_tags['hostname'] = $device['hostname'];
foreach ($tags as $k => $v) {
$v = preg_replace(['/ /', '/,/', '/=/'], ['\ ', '\,', '\='], $v);
if (empty($v)) {
$v = '_blank_';
}
$tmp_tags[$k] = $v;
}
foreach ($fields as $k => $v) {
if ($k == 'time') {
$k = 'rtime';
}
if (($value = $this->forceType($v)) !== null) {
$tmp_fields[$k] = $value;
}
}
if (empty($tmp_fields)) {
Log::warning('All fields empty, skipping update', ['orig_fields' => $fields]);
return;
}
Log::debug('InfluxDB data: ', [
'measurement' => $measurement,
'tags' => $tmp_tags,
'fields' => $tmp_fields,
]);
try {
$points = [
new \InfluxDB\Point(
$measurement,
null, // the measurement value
$tmp_tags,
$tmp_fields // optional additional fields
)
];
$this->connection->writePoints($points);
$this->recordStatistic($stat->end());
} catch (\Exception $e) {
Log::error('InfluxDB exception: ' . $e->getMessage());
Log::debug($e->getTraceAsString());
}
}
/**
* Create a new client and select the database
*
* @return \InfluxDB\Database
*/
public static function createFromConfig()
{
$host = Config::get('influxdb.host', 'localhost');
$transport = Config::get('influxdb.transport', 'http');
$port = Config::get('influxdb.port', 8086);
$db = Config::get('influxdb.db', 'librenms');
$username = Config::get('influxdb.username', '');
$password = Config::get('influxdb.password', '');
$timeout = Config::get('influxdb.timeout', 0);
$verify_ssl = Config::get('influxdb.verifySSL', false);
$client = new Client($host, $port, $username, $password, $transport == 'https', $verify_ssl, $timeout, $timeout);
if ($transport == 'udp') {
$client->setDriver(new UDP($host, $port));
}
return $client->selectDB($db);
}
private function forceType($data)
{
/*
* It is not trivial to detect if something is a float or an integer, and
* therefore may cause breakages on inserts.
* Just setting every number to a float gets around this, but may introduce
* inefficiencies.
* I've left the detection statement in there for a possible change in future,
* but currently everything just gets set to a float.
*/
if (is_numeric($data)) {
// If it is an Integer
if (ctype_digit($data)) {
return floatval($data);
// Else it is a float
} else {
return floatval($data);
}
}
return $data === 'U' ? null : $data;
}
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags()
{
return false;
}
}

View File

@ -0,0 +1,145 @@
<?php
/**
* OpenTSDB.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @copyright 2017 Yacine Benamsili <https://github.com/yac01/librenms.git>
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use Carbon\Carbon;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use Log;
class OpenTSDB extends BaseDatastore
{
/** @var \Socket\Raw\Socket $connection */
protected $connection;
public function __construct(\Socket\Raw\Factory $socketFactory)
{
parent::__construct();
$host = Config::get('opentsdb.host');
$port = Config::get('opentsdb.port', 2181);
try {
$this->connection = $socketFactory->createClient("$host:$port");
} catch (\Socket\Raw\Exception $e) {
Log::debug('OpenTSDB Error: ' . $e->getMessage());
}
if ($this->connection) {
Log::notice('Connected to OpenTSDB');
} else {
Log::error('Connection to OpenTSDB has failed!');
}
}
public function getName()
{
return 'OpenTSDB';
}
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields)
{
if (!$this->connection) {
Log::error("OpenTSDB Error: not connected\n");
return;
}
$flag = Config::get('opentsdb.co');
$timestamp = Carbon::now()->timestamp;
$tmp_tags = "hostname=".$device['hostname'];
foreach ($tags as $k => $v) {
$v = str_replace(array(' ',',','='), '_', $v);
if (!empty($v)) {
$tmp_tags = $tmp_tags ." ". $k ."=".$v;
}
}
if ($measurement == 'ports') {
foreach ($fields as $k => $v) {
$measurement = $k;
if ($flag == true) {
$measurement = $measurement.".".$device['co'];
}
$this->putData('port.'.$measurement, $timestamp, $v, $tmp_tags);
}
} else {
if ($flag == true) {
$measurement = $measurement . '.' . $device['co'];
}
foreach ($fields as $k => $v) {
$tmp_tags_key = $tmp_tags . " " . "key" . "=" . $k;
$this->putData($measurement, $timestamp, $v, $tmp_tags_key);
}
}
}
private function putData($measurement, $timestamp, $value, $tags)
{
try {
$stat = Measurement::start('put');
$line = sprintf('put net.%s %d %f %s', strtolower($measurement), $timestamp, $value, $tags);
Log::debug("Sending to OpenTSDB: $line\n");
$this->connection->write("$line\n"); // send $line into OpenTSDB
$this->recordStatistic($stat->end());
} catch (\Socket\Raw\Exception $e) {
Log::error('OpenTSDB Error: ' . $e->getMessage());
}
}
public static function isEnabled()
{
return Config::get('opentsdb.enable', false);
}
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags()
{
return false;
}
}

View File

@ -0,0 +1,132 @@
<?php
/**
* Prometheus.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @copyright 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use GuzzleHttp\Exception\GuzzleException;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use Log;
class Prometheus extends BaseDatastore
{
private $client;
private $base_uri;
private $default_opts;
private $enabled;
public function __construct(\GuzzleHttp\Client $client)
{
parent::__construct();
$this->client = $client;
$url = Config::get('prometheus.url');
$job = Config::get('prometheus.job', 'librenms');
$this->base_uri = "$url/metrics/job/$job/instance/";
$this->default_opts = [
'headers' => ['Content-Type' => 'text/plain'],
];
if ($proxy = get_proxy()) {
$this->default_opts['proxy'] = $proxy;
}
$this->enabled = self::isEnabled();
}
public function getName()
{
return 'Prometheus';
}
public static function isEnabled()
{
return Config::get('prometheus.enable', false);
}
public function put($device, $measurement, $tags, $fields)
{
$stat = Measurement::start('put');
// skip if needed
if (!$this->enabled) {
return;
}
try {
$vals = "";
$promtags = "/measurement/" . $measurement;
foreach ($fields as $k => $v) {
if ($v !== null) {
$vals .= "$k $v\n";
}
}
foreach ($tags as $t => $v) {
if ($v !== null) {
$promtags .= "/$t/$v";
}
}
$options = $this->getDefaultOptions();
$options['body'] = $vals;
$promurl = $this->base_uri . $device['hostname'] . $promtags;
$promurl = str_replace(" ", "-", $promurl); // Prometheus doesn't handle tags with spaces in url
Log::debug("Prometheus put $promurl: ", [
'measurement' => $measurement,
'tags' => $tags,
'fields' => $fields,
'vals' => $vals,
]);
$result = $this->client->request('POST', $promurl, $options);
$this->recordStatistic($stat->end());
if ($result->getStatusCode() !== 200) {
Log::error('Prometheus Error: ' . $result->getReasonPhrase());
}
} catch (GuzzleException $e) {
Log::error("Prometheus Exception: " . $e->getMessage());
}
}
private function getDefaultOptions()
{
return $this->default_opts;
}
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags()
{
return false;
}
}

496
LibreNMS/Data/Store/Rrd.php Normal file
View File

@ -0,0 +1,496 @@
<?php
/**
* Rrd.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use LibreNMS\Exceptions\FileExistsException;
use LibreNMS\Proc;
use Log;
class Rrd extends BaseDatastore
{
private $disabled = false;
/** @var Proc $sync_process */
private $sync_process;
/** @var Proc $async_process */
private $async_process;
private $rrd_dir;
private $version;
private $rrdcached;
private $rra;
private $step;
public function __construct()
{
parent::__construct();
$this->rrdcached = Config::get('rrdcached', false);
$this->init();
$this->rrd_dir = Config::get('rrd_dir', Config::get('install_dir') . '/rrd');
$this->step = Config::get('rrd.step', 300);
$this->rra = Config::get(
'rrd_rra',
'RRA:AVERAGE:0.5:1:2016 RRA:AVERAGE:0.5:6:1440 RRA:AVERAGE:0.5:24:1440 RRA:AVERAGE:0.5:288:1440 ' .
' RRA:MIN:0.5:1:720 RRA:MIN:0.5:6:1440 RRA:MIN:0.5:24:775 RRA:MIN:0.5:288:797 ' .
' RRA:MAX:0.5:1:720 RRA:MAX:0.5:6:1440 RRA:MAX:0.5:24:775 RRA:MAX:0.5:288:797 ' .
' RRA:LAST:0.5:1:1440 '
);
$this->version = Config::get('rrdtool_version', '1.4');
}
public function getName()
{
return 'RRD';
}
public static function isEnabled()
{
return Config::get('rrd.enable', true);
}
/**
* Opens up a pipe to RRDTool using handles provided
*
* @param bool $dual_process start an additional process that's output should be read after every command
* @return bool the process(s) have been successfully started
*/
public function init($dual_process = true)
{
$command = Config::get('rrdtool', 'rrdtool') . ' -';
$descriptor_spec = [
0 => ['pipe', 'r'], // stdin is a pipe that the child will read from
1 => ['pipe', 'w'], // stdout is a pipe that the child will write to
2 => ['pipe', 'w'], // stderr is a pipe that the child will write to
];
$cwd = Config::get('rrd_dir');
if (!$this->isSyncRunning()) {
$this->sync_process = new Proc($command, $descriptor_spec, $cwd);
}
if ($dual_process && !$this->isAsyncRunning()) {
$this->async_process = new Proc($command, $descriptor_spec, $cwd);
$this->async_process->setSynchronous(false);
}
return $this->isSyncRunning() && ($dual_process ? $this->isAsyncRunning() : true);
}
public function isSyncRunning()
{
return isset($this->sync_process) && $this->sync_process->isRunning();
}
public function isAsyncRunning()
{
return isset($this->async_process) && $this->async_process->isRunning();
}
/**
* Close all open rrdtool processes.
* This should be done before exiting
*/
public function close()
{
if ($this->isSyncRunning()) {
$this->sync_process->close('quit');
}
if ($this->isAsyncRunning()) {
$this->async_process->close('quit');
}
}
/**
* rrdtool backend implementation of data_update
*
* Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device device array
* @param string $measurement the name of this measurement (if no rrd_name tag is given, this will be used to name the file)
* @param array $tags tags to pass additional info to rrdtool
* @param array $fields data values to update
*/
public function put($device, $measurement, $tags, $fields)
{
$rrd_name = isset($tags['rrd_name']) ? $tags['rrd_name'] : $measurement;
$step = isset($tags['rrd_step']) ? $tags['rrd_step'] : $this->step;
if (!empty($tags['rrd_oldname'])) {
self::renameFile($device, $tags['rrd_oldname'], $rrd_name);
}
if (isset($tags['rrd_proxmox_name'])) {
$pmxvars = $tags['rrd_proxmox_name'];
$rrd = self::proxmoxName($pmxvars['pmxcluster'], $pmxvars['vmid'], $pmxvars['vmport']);
} else {
$rrd = self::name($device['hostname'], $rrd_name);
}
if (isset($tags['rrd_def'])) {
$rrd_def = $tags['rrd_def'];
// filter out data not in the definition
$fields = array_filter($fields, function ($key) use ($rrd_def) {
$valid = $rrd_def->isValidDataset($key);
if (!$valid) {
Log::warning("RRD warning: unused data sent $key");
}
return $valid;
}, ARRAY_FILTER_USE_KEY);
if (!$this->checkRrdExists($rrd)) {
$newdef = "--step $step $rrd_def $this->rra";
$this->command('create', $rrd, $newdef);
}
}
$this->update($rrd, $fields);
}
/**
* Updates an rrd database at $filename using $options
* Where $options is an array, each entry which is not a number is replaced with "U"
*
* @internal
* @param string $filename
* @param array $data
* @return array|string
*/
public function update($filename, $data)
{
$values = [];
// Do some sanitation on the data if passed as an array.
if (is_array($data)) {
$values[] = 'N';
foreach ($data as $v) {
if (!is_numeric($v)) {
$v = 'U';
}
$values[] = $v;
}
$data = implode(':', $values);
return $this->command('update', $filename, $data);
} else {
return 'Bad options passed to rrdtool_update';
}
} // rrdtool_update
/**
* Modify an rrd file's max value and trim the peaks as defined by rrdtool
*
* @param string $type only 'port' is supported at this time
* @param string $filename the path to the rrd file
* @param integer $max the new max value
* @return bool
*/
public function tune($type, $filename, $max)
{
$fields = [];
if ($type === 'port') {
if ($max < 10000000) {
return false;
}
$max = $max / 8;
$fields = [
'INOCTETS',
'OUTOCTETS',
'INERRORS',
'OUTERRORS',
'INUCASTPKTS',
'OUTUCASTPKTS',
'INNUCASTPKTS',
'OUTNUCASTPKTS',
'INDISCARDS',
'OUTDISCARDS',
'INUNKNOWNPROTOS',
'INBROADCASTPKTS',
'OUTBROADCASTPKTS',
'INMULTICASTPKTS',
'OUTMULTICASTPKTS'
];
}
if (count($fields) > 0) {
$options = "--maximum " . implode(":$max --maximum ", $fields) . ":$max";
$this->command('tune', $filename, $options);
}
return true;
} // rrdtool_tune
/**
* Generates a filename for a proxmox cluster rrd
*
* @param $pmxcluster
* @param $vmid
* @param $vmport
* @return string full path to the rrd.
*/
public function proxmoxName($pmxcluster, $vmid, $vmport)
{
$pmxcdir = join('/', [$this->rrd_dir, 'proxmox', self::safeName($pmxcluster)]);
// this is not needed for remote rrdcached
if (!is_dir($pmxcdir)) {
mkdir($pmxcdir, 0775, true);
}
return join('/', [$pmxcdir, self::safeName($vmid . '_netif_' . $vmport . '.rrd')]);
}
/**
* rename an rrdfile, can only be done on the LibreNMS server hosting the rrd files
*
* @param array $device Device object
* @param string|array $oldname RRD name array as used with rrd_name()
* @param string|array $newname RRD name array as used with rrd_name()
* @return bool indicating rename success or failure
*/
public function renameFile($device, $oldname, $newname)
{
$oldrrd = self::name($device['hostname'], $oldname);
$newrrd = self::name($device['hostname'], $newname);
if (is_file($oldrrd) && !is_file($newrrd)) {
if (rename($oldrrd, $newrrd)) {
log_event("Renamed $oldrrd to $newrrd", $device, "poller", 1);
return true;
} else {
log_event("Failed to rename $oldrrd to $newrrd", $device, "poller", 5);
return false;
}
} else {
// we don't need to rename the file
return true;
}
}
/**
* Generates a filename based on the hostname (or IP) and some extra items
*
* @param string $host Host name
* @param array|string $extra Components of RRD filename - will be separated with "-", or a pre-formed rrdname
* @param string $extension File extension (default is .rrd)
* @return string the name of the rrd file for $host's $extra component
*/
public function name($host, $extra, $extension = ".rrd")
{
$filename = self::safeName(is_array($extra) ? implode("-", $extra) : $extra);
return implode("/", [$this->dirFromHost($host), $filename . $extension]);
}
/**
* Generates a path based on the hostname (or IP)
*
* @param string $host Host name
* @return string the name of the rrd directory for $host
*/
public function dirFromHost($host)
{
$host = str_replace(':', '_', trim($host, '[]'));
return implode("/", [$this->rrd_dir, $host]);
}
/**
* Generates and pipes a command to rrdtool
*
* @internal
* @param string $command create, update, updatev, graph, graphv, dump, restore, fetch, tune, first, last, lastupdate, info, resize, xport, flushcached
* @param string $filename The full patth to the rrd file
* @param string $options rrdtool command options
* @return array the output of stdout and stderr in an array
* @throws \Exception thrown when the rrdtool process(s) cannot be started
*/
private function command($command, $filename, $options)
{
global $vdebug;
$stat = Measurement::start($this->coalesceStatisticType($command));
try {
$cmd = self::buildCommand($command, $filename, $options);
} catch (FileExistsException $e) {
Log::debug("RRD[%g$filename already exists%n]", ['color' => true]);
return [null, null];
}
Log::debug("RRD[%g$cmd%n]", ['color' => true]);
// do not write rrd files, but allow read-only commands
$ro_commands = ['graph', 'graphv', 'dump', 'fetch', 'first', 'last', 'lastupdate', 'info', 'xport'];
if ($this->disabled && !in_array($command, $ro_commands)) {
if (!Config::get('hide_rrd_disabled')) {
Log::debug('[%rRRD Disabled%n]', ['color' => true]);
}
return [null, null];
}
// send the command!
if ($command == 'last' && $this->init(false)) {
// send this to our synchronous process so output is guaranteed
$output = $this->sync_process->sendCommand($cmd);
} elseif ($this->init()) {
// don't care about the return of other commands, so send them to the faster async process
$output = $this->async_process->sendCommand($cmd);
} else {
Log::error('rrdtool could not start');
}
if ($vdebug) {
echo 'RRDtool Output: ';
echo $output[0];
echo $output[1];
}
$this->recordStatistic($stat->end());
return $output;
}
/**
* Build a command for rrdtool
* Shortens the filename as needed
* Determines if --daemon and -O should be used
*
* @internal
* @param string $command The base rrdtool command. Usually create, update, last.
* @param string $filename The full path to the rrd file
* @param string $options Options for the command possibly including the rrd definition
* @return string returns a full command ready to be piped to rrdtool
* @throws FileExistsException if rrdtool <1.4.3 and the rrd file exists locally
*/
private function buildCommand($command, $filename, $options)
{
if ($command == 'create') {
// <1.4.3 doesn't support -O, so make sure the file doesn't exist
if (version_compare($this->version, '1.4.3', '<')) {
if (is_file($filename)) {
throw new FileExistsException();
}
} else {
$options .= ' -O';
}
}
// no remote for create < 1.5.5 and tune < 1.5
if ($this->rrdcached &&
!($command == 'create' && version_compare($this->version, '1.5.5', '<')) &&
!($command == 'tune' && $this->rrdcached && version_compare($this->version, '1.5', '<'))
) {
// only relative paths if using rrdcached
$filename = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $filename);
$options = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $options);
return "$command $filename $options --daemon " . $this->rrdcached;
}
return "$command $filename $options";
}
/**
* Checks if the rrd file exists on the server
* This will perform a remote check if using rrdcached and rrdtool >= 1.5
*
* @param string $filename full path to the rrd file
* @return bool whether or not the passed rrd file exists
*/
public function checkRrdExists($filename)
{
if ($this->rrdcached && version_compare($this->version, '1.5', '>=')) {
$chk = $this->command('last', $filename, '');
$filename = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $filename);
return !str_contains(implode($chk), "$filename': No such file or directory");
} else {
return is_file($filename);
}
}
/**
* Generates a graph file at $graph_file using $options
* Opens its own rrdtool pipe.
*
* @param string $graph_file
* @param string $options
* @return integer
*/
public function graph($graph_file, $options)
{
if ($this->init(false)) {
$cmd = $this->buildCommand('graph', $graph_file, $options);
$output = implode($this->sync_process->sendCommand($cmd));
d_echo("<p>$cmd</p>\n<p>command returned ($output)</p>");
return $output;
} else {
return 0;
}
}
public function __destruct()
{
$this->close();
}
/**
* Remove invalid characters from the rrd file name
*
* @param string $name
* @return string
*/
public static function safeName($name)
{
return (string)preg_replace('/[^a-zA-Z0-9,._\-]/', '_', $name);
}
/**
* Remove invalid characters from the rrd description
*
* @param string $descr
* @return string
*/
public static function safeDescr($descr)
{
return (string)preg_replace('/[^a-zA-Z0-9,._\-\/\ ]/', ' ', $descr);
}
/**
* Only track update and create primarily, just put all others in an "other" bin
*
* @param $type
* @return string
*/
private function coalesceStatisticType($type)
{
return ($type == 'update' || $type == 'create') ? $type : 'other';
}
}

View File

@ -0,0 +1,74 @@
<?php
/**
* Datastore.php
*
* Interface for datastores. Will be used to send them data through the put() method
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Interfaces\Data;
interface Datastore
{
/**
* Check if this is enabled by the configuration
*
* @return boolean
*/
public static function isEnabled();
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags();
/**
* The name of this datastore
*
* @return string
*/
public function getName();
/**
* Array of stats should be [type => [count => n, time => s]]
*
* @return array
*/
public function getStats();
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields);
}

View File

@ -30,8 +30,9 @@ use LibreNMS\Exceptions\InvalidRrdTypeException;
class RrdDefinition
{
private static $types = array('GAUGE', 'DERIVE', 'COUNTER', 'ABSOLUTE', 'DCOUNTER', 'DDERIVE');
private $dataSets = array();
private static $types = ['GAUGE', 'DERIVE', 'COUNTER', 'ABSOLUTE', 'DCOUNTER', 'DDERIVE'];
private $dataSets = [];
private $skipNameCheck = false;
/**
* Make a new empty RrdDefinition
@ -58,14 +59,14 @@ class RrdDefinition
d_echo("DS must be set to a non-empty string.");
}
$ds = array();
$ds[] = $this->escapeName($name);
$ds[] = $this->checkType($type);
$ds[] = is_null($heartbeat) ? Config::get('rrd.heartbeat') : $heartbeat;
$ds[] = is_null($min) ? 'U' : $min;
$ds[] = is_null($max) ? 'U' : $max;
$this->dataSets[] = $ds;
$name = $this->escapeName($name);
$this->dataSets[$name] = [
$name,
$this->checkType($type),
is_null($heartbeat) ? Config::get('rrd.heartbeat') : $heartbeat,
is_null($min) ? 'U' : $min,
is_null($max) ? 'U' : $max,
];
return $this;
}
@ -82,6 +83,22 @@ class RrdDefinition
}, '');
}
/**
* Check if the give dataset name is valid for this definition
*
* @param $name
* @return bool
*/
public function isValidDataset($name)
{
return $this->skipNameCheck || isset($this->dataSets[$name]);
}
public function disableNameChecking()
{
$this->skipNameCheck = true;
}
/**
* Check that the data set type is valid.
*

View File

@ -367,7 +367,10 @@ GRAPHQL;
}
if (!isset($previous_release['published_at'])) {
throw new Exception("Could not find previous release tag.");
throw new Exception(
$previous_release['message'] ??
"Could not find previous release tag. ($this->from)"
);
}
$this->getPullRequests($previous_release['published_at']);

View File

@ -58,7 +58,7 @@ class RrdCheck extends BaseValidation
$screenpad = 0;
foreach ($rrd_iterator as $filename => $file) {
$rrd_test_result = rrdtest($filename, $output, $error);
$rrd_test_result = $this->test($filename, $output, $error);
$loopcount++;
if (($loopcount % 50) == 0) {
@ -80,4 +80,44 @@ class RrdCheck extends BaseValidation
echo "\033[" . $screenpad . "D";
echo "Status: " . $loopcount . "/" . $rrd_total . " - Complete\n";
}
/**
* Run rrdtool info on a file path
*
* @param string $path Path to pass to rrdtool info
* @param string $stdOutput Variable to recieve the output of STDOUT
* @param string $stdError Variable to recieve the output of STDERR
*
* @return int exit code
*
**/
private function test($path, &$stdOutput, &$stdError)
{
//rrdtool info <escaped rrd path>
$command = Config::get('rrdtool') . ' info ' . escapeshellarg($path);
$process = proc_open(
$command,
array (
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('pipe', 'w'),
),
$pipes
);
if (!is_resource($process)) {
throw new \RuntimeException('Could not create a valid process');
}
$status = proc_get_status($process);
while ($status['running']) {
usleep(2000); // Sleep 2000 microseconds or 2 milliseconds
$status = proc_get_status($process);
}
$stdOutput = stream_get_contents($pipes[1]);
$stdError = stream_get_contents($pipes[2]);
proc_close($process);
return $status['exitcode'];
}
}

36
app/Facades/Rrd.php Normal file
View File

@ -0,0 +1,36 @@
<?php
/**
* Rrd.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace App\Facades;
use Illuminate\Support\Facades\Facade;
class Rrd extends Facade
{
protected static function getFacadeAccessor()
{
return 'LibreNMS\Data\Store\Rrd';
}
}

View File

@ -260,7 +260,7 @@ class PingCheck implements ShouldQueue
$rules->runRules($device->device_id);
// add data to rrd
data_update($device->toArray(), 'ping-perf', $this->rrd_tags, ['ping' => $device->last_ping_timetaken]);
app('Datastore')->put($device->toArray(), 'ping-perf', $this->rrd_tags, ['ping' => $device->last_ping_timetaken]);
// done with this device
$this->complete($device->hostname);

View File

@ -0,0 +1,74 @@
<?php
/**
* DatastoreServiceProvider.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace App\Providers;
use Illuminate\Contracts\Foundation\Application;
use Illuminate\Support\ServiceProvider;
use LibreNMS\Data\Store\Datastore;
use LibreNMS\Interfaces\Data\Datastore as DatastoreContract;
class DatastoreServiceProvider extends ServiceProvider
{
protected $namespace = 'LibreNMS\\Data\\Store\\';
protected $stores = [
'LibreNMS\Data\Store\Graphite',
'LibreNMS\Data\Store\InfluxDB',
'LibreNMS\Data\Store\OpenTSDB',
'LibreNMS\Data\Store\Prometheus',
'LibreNMS\Data\Store\Rrd',
];
public function register()
{
// set up bindings
foreach ($this->stores as $store) {
$this->app->singleton($store);
}
// bind the Datastore object
$this->app->singleton('Datastore', function (Application $app, $options) {
// only tag datastores enabled by config
$stores = array_filter($this->stores, function ($store) {
/** @var DatastoreContract $store */
return $store::isEnabled();
});
$app->tag($stores, ['datastore']);
return new Datastore(iterator_to_array($app->tagged('datastore')));
});
// additional bindings
$this->registerInflux();
}
public function registerInflux()
{
$this->app->singleton('InfluxDB\Database', function ($app) {
return \LibreNMS\Data\Store\InfluxDB::createFromConfig();
});
}
}

View File

@ -13,31 +13,19 @@
* the source code distribution for details.
*/
use LibreNMS\Data\Store\Datastore;
$init_modules = array();
require __DIR__ . '/includes/init.php';
$options = getopt('d::h:f:;');
$options = getopt('drfpgh:');
if (set_debug(isset($options['d']))) {
echo "DEBUG!\n";
}
if (isset($options['f'])) {
\LibreNMS\Config::set('noinfluxdb', true);
}
if (isset($options['p'])) {
$prometheus = false;
}
if (\LibreNMS\Config::get('noinfluxdb') !== true && \LibreNMS\Config::get('influxdb.enable') === true) {
$influxdb = influxdb_connect();
} else {
$influxdb = false;
}
$poller_start = microtime(true);
rrdtool_initialize();
$datastore = Datastore::init($options);
echo "Starting service polling run:\n\n";
$polled_services = 0;
@ -96,4 +84,4 @@ $string = $argv[0] . " " . date(\LibreNMS\Config::get('dateformat.compact'))
." - $polled_services services polled in $poller_time secs";
d_echo("$string\n");
rrdtool_close();
Datastore::terminate();

View File

@ -30,6 +30,7 @@
"ext-session": "*",
"ext-xml": "*",
"amenadiel/jpgraph": "^3.6",
"clue/socket-raw": "^1.4",
"dapphp/radius": "^2.0",
"darkghosthunter/larapoke": "^3.1",
"doctrine/dbal": "^2.9",

57
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "af1e9508013c6187a1ebaf4beaf59223",
"content-hash": "206e47998ed4d125251600e26fe67462",
"packages": [
{
"name": "amenadiel/jpgraph",
@ -61,6 +61,61 @@
],
"time": "2018-10-14T21:43:30+00:00"
},
{
"name": "clue/socket-raw",
"version": "v1.4.1",
"source": {
"type": "git",
"url": "https://github.com/clue/php-socket-raw.git",
"reference": "00ab102d061f6cdb895e79dd4d69140c7bda31cc"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/clue/php-socket-raw/zipball/00ab102d061f6cdb895e79dd4d69140c7bda31cc",
"reference": "00ab102d061f6cdb895e79dd4d69140c7bda31cc",
"shasum": ""
},
"require": {
"ext-sockets": "*",
"php": ">=5.3"
},
"require-dev": {
"phpunit/phpunit": "^7.0 || ^6.0 || ^5.2 || ^4.8.35"
},
"type": "library",
"autoload": {
"psr-4": {
"Socket\\Raw\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Christian Lück",
"email": "christian@clue.engineering"
}
],
"description": "Simple and lightweight OOP wrapper for PHP's low-level sockets extension (ext-sockets)",
"homepage": "https://github.com/clue/php-socket-raw",
"keywords": [
"Socket",
"client",
"datagram",
"dgram",
"icmp",
"ipv6",
"server",
"stream",
"tcp",
"udg",
"udp",
"unix"
],
"time": "2019-10-28T12:32:07+00:00"
},
{
"name": "dapphp/radius",
"version": "2.5.4",

View File

@ -180,6 +180,7 @@ return [
App\Providers\RouteServiceProvider::class,
App\Providers\ComposerServiceProvider::class,
App\Providers\ViewServiceProvider::class,
App\Providers\DatastoreServiceProvider::class,
/*
* LibreNMS Service Providers...
@ -241,6 +242,7 @@ return [
// LibreNMS
'Permissions' => \App\Facades\Permissions::class,
'DeviceCache' => \App\Facades\DeviceCache::class,
'Rrd' => App\Facades\Rrd::class,
],
];

View File

@ -9,6 +9,9 @@
* @copyright (C) 2006 - 2012 Adam Armstrong
*/
use LibreNMS\Authentication\LegacyAuth;
use LibreNMS\Data\Store\Datastore;
$start = microtime(true);
$init_modules = array('web', 'graphs', 'auth');
@ -22,11 +25,9 @@ if (!$auth) {
set_debug(isset($_GET['debug']));
rrdtool_initialize(false);
require \LibreNMS\Config::get('install_dir') . '/includes/html/graphs/graph.inc.php';
rrdtool_close();
Datastore::terminate();
if ($debug) {
echo '<br />';

View File

@ -436,7 +436,7 @@ function getidbyname($hostname)
function safename($name)
{
return preg_replace('/[^a-zA-Z0-9,._\-]/', '_', $name);
return \LibreNMS\Data\Store\Rrd::safeName($name);
}
/**
@ -446,7 +446,7 @@ function safename($name)
*/
function safedescr($descr)
{
return preg_replace('/[^a-zA-Z0-9,._\-\/\ ]/', ' ', $descr);
return \LibreNMS\Data\Store\Rrd::safeDescr($descr);
}
function zeropad($num, $length = 2)

View File

@ -18,25 +18,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Filter all elements with keys that start with 'rrd_'
*
* @param array $arr input array
* @return array Copy of $arr with all keys beginning with 'rrd_' removed.
*/
function rrd_array_filter($arr)
{
$result = array();
foreach ($arr as $k => $v) {
if (strpos($k, 'rrd_') === 0) {
continue;
}
$result[$k] = $v;
}
return $result;
} // rrd_array_filter
/**
* Datastore-independent function which should be used for all polled metrics.
*
@ -54,20 +35,6 @@ function rrd_array_filter($arr)
*/
function data_update($device, $measurement, $tags, $fields)
{
// convenience conversion to allow calling with a single value, so, e.g., these are equivalent:
// data_update($device, 'mymeasurement', $tags, 1234);
// AND
// data_update($device, 'mymeasurement', $tags, array('mymeasurement' => 1234));
if (!is_array($fields)) {
$fields = array($measurement => $fields);
}
// rrdtool_data_update() will only use the tags it deems relevant, so we pass all of them.
// However, influxdb saves all tags, so we filter out the ones beginning with 'rrd_'.
rrdtool_data_update($device, $measurement, $tags, $fields);
prometheus_push($device, $measurement, rrd_array_filter($tags), $fields);
influx_update($device, $measurement, rrd_array_filter($tags), $fields);
opentsdb_update($device, $measurement, rrd_array_filter($tags), $fields);
graphite_update($device, $measurement, $tags, $fields);
} // data_update
$datastore = app('Datastore');
$datastore->put($device, $measurement, $tags, $fields);
}

View File

@ -344,26 +344,6 @@ function percent_colour($perc)
return sprintf('#%02x%02x%02x', $r, $b, $b);
}
// Returns the last in/out errors value in RRD
function interface_errors($rrd_file, $period = '-1d')
{
$errors = array();
$cmd = Config::get('rrdtool') . " fetch -s $period -e -300s $rrd_file AVERAGE | grep : | cut -d\" \" -f 4,5";
$data = trim(shell_exec($cmd));
$in_errors = 0;
$out_errors = 0;
foreach (explode("\n", $data) as $entry) {
list($in, $out) = explode(" ", $entry);
$in_errors += ($in * 300);
$out_errors += ($out * 300);
}
$errors['in'] = round($in_errors);
$errors['out'] = round($out_errors);
return $errors;
}
/**
* @param $device
* @return string the logo image path for this device. Images are often wide, not square.
@ -1542,30 +1522,6 @@ function function_check($function)
return function_exists($function);
}
function force_influx_data($data)
{
/*
* It is not trivial to detect if something is a float or an integer, and
* therefore may cause breakages on inserts.
* Just setting every number to a float gets around this, but may introduce
* inefficiencies.
* I've left the detection statement in there for a possible change in future,
* but currently everything just gets set to a float.
*/
if (is_numeric($data)) {
// If it is an Integer
if (ctype_digit($data)) {
return floatval($data);
// Else it is a float
} else {
return floatval($data);
}
} else {
return $data;
}
}// end force_influx_data
/**
* Try to determine the address family (IPv4 or IPv6) associated with an SNMP
* transport specifier (like "udp", "udp6", etc.).
@ -1662,50 +1618,6 @@ function dnslookup($device, $type = false, $return = false)
return $record[0][$return];
}//end dnslookup
/**
* Run rrdtool info on a file path
*
* @param string $path Path to pass to rrdtool info
* @param string $stdOutput Variable to recieve the output of STDOUT
* @param string $stdError Variable to recieve the output of STDERR
*
* @return int exit code
*
**/
function rrdtest($path, &$stdOutput, &$stdError)
{
//rrdtool info <escaped rrd path>
$command = Config::get('rrdtool') . ' info ' . escapeshellarg($path);
$process = proc_open(
$command,
array (
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('pipe', 'w'),
),
$pipes
);
if (!is_resource($process)) {
throw new \RuntimeException('Could not create a valid process');
}
$status = proc_get_status($process);
while ($status['running']) {
usleep(2000); // Sleep 2000 microseconds or 2 milliseconds
$status = proc_get_status($process);
}
$stdOutput = stream_get_contents($pipes[1]);
$stdError = stream_get_contents($pipes[2]);
proc_close($process);
return $status['exitcode'];
}
/**
* Create a new state index. Update translations if $states is given.
*
@ -2062,10 +1974,9 @@ function get_toner_levels($device, $raw_value, $capacity)
*/
function initStats()
{
global $snmp_stats, $rrd_stats;
global $snmp_stats_last, $rrd_stats_last;
global $snmp_stats, $snmp_stats_last;
if (!isset($snmp_stats, $rrd_stats)) {
if (!isset($snmp_stats)) {
$snmp_stats = array(
'ops' => array(
'snmpget' => 0,
@ -2079,20 +1990,6 @@ function initStats()
)
);
$snmp_stats_last = $snmp_stats;
$rrd_stats = array(
'ops' => array(
'update' => 0,
'create' => 0,
'other' => 0,
),
'time' => array(
'update' => 0.0,
'create' => 0.0,
'other' => 0.0,
),
);
$rrd_stats_last = $rrd_stats;
}
}
@ -2103,25 +2000,29 @@ function initStats()
*/
function printChangedStats($update_only = false)
{
global $snmp_stats, $db_stats, $rrd_stats;
global $snmp_stats_last, $db_stats_last, $rrd_stats_last;
global $snmp_stats, $db_stats;
global $snmp_stats_last, $db_stats_last;
$output = sprintf(
">> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs]",
array_sum($snmp_stats['ops']) - array_sum($snmp_stats_last['ops']),
array_sum($snmp_stats['time']) - array_sum($snmp_stats_last['time']),
array_sum($db_stats['ops']) - array_sum($db_stats_last['ops']),
array_sum($db_stats['time']) - array_sum($db_stats_last['time'])
);
foreach (app('Datastore')->getStats() as $datastore => $stats) {
/** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */
$output .= sprintf(" %s: [%d/%.2fs]", $datastore, $stats->getCountDiff(), $stats->getDurationDiff());
$stats->checkpoint();
}
if (!$update_only) {
printf(
">> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs] RRD: [%d/%.2fs]\n",
array_sum($snmp_stats['ops']) - array_sum($snmp_stats_last['ops']),
array_sum($snmp_stats['time']) - array_sum($snmp_stats_last['time']),
array_sum($db_stats['ops']) - array_sum($db_stats_last['ops']),
array_sum($db_stats['time']) - array_sum($db_stats_last['time']),
array_sum($rrd_stats['ops']) - array_sum($rrd_stats_last['ops']),
array_sum($rrd_stats['time']) - array_sum($rrd_stats_last['time'])
);
echo $output . PHP_EOL;
}
// make a new checkpoint
$snmp_stats_last = $snmp_stats;
$db_stats_last = $db_stats;
$rrd_stats_last = $rrd_stats;
}
/**
@ -2129,7 +2030,7 @@ function printChangedStats($update_only = false)
*/
function printStats()
{
global $snmp_stats, $db_stats, $rrd_stats;
global $snmp_stats, $db_stats;
if ($snmp_stats) {
printf(
@ -2167,42 +2068,18 @@ function printStats()
);
}
if ($rrd_stats) {
printf(
"RRD [%d/%.2fs]: Update[%d/%.2fs] Create [%d/%.2fs] Other[%d/%.2fs]\n",
array_sum($rrd_stats['ops']),
array_sum($rrd_stats['time']),
$rrd_stats['ops']['update'],
$rrd_stats['time']['update'],
$rrd_stats['ops']['create'],
$rrd_stats['time']['create'],
$rrd_stats['ops']['other'],
$rrd_stats['time']['other']
);
foreach (app('Datastore')->getStats() as $datastore => $stats) {
/** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */
printf("%s [%d/%.2fs]:", $datastore, $stats->getTotalCount(), $stats->getTotalDuration());
foreach ($stats as $stat) {
/** @var \LibreNMS\Data\Measure\MeasurementSummary $stat */
printf(" %s[%d/%.2fs]", ucfirst($stat->getType()), $stat->getCount(), $stat->getDuration());
}
echo PHP_EOL;
}
}
/**
* Update statistics for rrd operations
*
* @param string $stat create, update, and other
* @param float $start_time The time the operation started with 'microtime(true)'
* @return float The calculated run time
*/
function recordRrdStatistic($stat, $start_time)
{
global $rrd_stats;
initStats();
$stat = ($stat == 'update' || $stat == 'create') ? $stat : 'other';
$runtime = microtime(true) - $start_time;
$rrd_stats['ops'][$stat]++;
$rrd_stats['time'][$stat] += $runtime;
return $runtime;
}
/**
* @param string $stat snmpget, snmpwalk
* @param float $start_time The time the operation started with 'microtime(true)'

View File

@ -1,50 +0,0 @@
<?php
/*
* LibreNMS
*
* Copyright (c) 2017 Falk Stern <https://github.com/fstern/ >
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version. Please see LICENSE.txt at the top level of
* the source code distribution for details.
*/
function graphite_update($device, $measurement, $tags, $fields)
{
global $graphite;
if ($graphite != false) {
$timestamp = time();
$graphite_prefix = \LibreNMS\Config::get('graphite.prefix');
// metrics will be built as prefix.hostname.measurement.field value timestamp
// metric fields can not contain . as this is used by graphite as a field separator
$hostname = preg_replace('/\./', '_', $device['hostname']);
$measurement = preg_replace(array('/\./', '/\//'), '_', $measurement);
$measurement = preg_replace('/\|/', '.', $measurement);
$measurement_name = preg_replace('/\./', '_', $tags['rrd_name']);
if (is_array($measurement_name)) {
$ms_name = implode(".", $measurement_name);
} else {
$ms_name = $measurement_name;
}
// remove the port-id tags from the metric
if (preg_match('/^port-id\d+/', $ms_name)) {
$ms_name = "";
}
foreach ($fields as $k => $v) {
// Send zero for fields without values
if (empty($v)) {
$v = 0;
}
$metric = implode(".", array_filter(array($graphite_prefix, $hostname, $measurement, $ms_name, $k)));
// Further sanitize the full metric before sending, whitespace isn't allowed
$metric = preg_replace('/\s+/', '_', $metric);
$line = implode(" ", array($metric, $v, $timestamp));
d_echo("Sending $line\n");
fwrite($graphite, $line . "\n");
}
}
}

View File

@ -21,6 +21,7 @@ use Illuminate\Routing\Router;
use Illuminate\Support\Facades\Validator;
use LibreNMS\Alerting\QueryBuilderParser;
use LibreNMS\Config;
use LibreNMS\Data\Store\Datastore;
use LibreNMS\Exceptions\InvalidIpException;
use LibreNMS\Util\IPv4;
@ -83,9 +84,8 @@ function api_get_graph(array $vars)
ob_start();
rrdtool_initialize(false);
include 'includes/html/graphs/graph.inc.php';
rrdtool_close();
Datastore::terminate();
$image = ob_get_contents();
ob_end_clean();

View File

@ -42,7 +42,7 @@ if (!empty($ifName) && is_numeric($port_id) && is_numeric($port_id)) {
($device_tune == "true" && $port_tune != 'false') ||
(\LibreNMS\Config::get('rrdtool_tune') == "true" && $port_tune != 'false' && $device_tune != 'false')) {
$rrdfile = get_port_rrdfile_path($device['hostname'], $port_id);
rrdtool_tune('port', $rrdfile, $speed);
Rrd::tune('port', $rrdfile, $speed);
}
}
$status = 'ok';

View File

@ -18,7 +18,7 @@
require 'includes/html/graphs/common.inc.php';
$proxmox_rrd = proxmox_rrd_name($vars['cluster'], $vars['vmid'], $vars['port']);
$proxmox_rrd = Rrd::proxmoxName($vars['cluster'], $vars['vmid'], $vars['port']);
if (rrdtool_check_rrd_exists($proxmox_rrd)) {
$rrd_filename = $proxmox_rrd;

View File

@ -1,89 +0,0 @@
<?php
use LibreNMS\Config;
/*
* LibreNMS
*
* Copyright (c) 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version. Please see LICENSE.txt at the top level of
* the source code distribution for details.
*/
function influxdb_connect()
{
$influxdb_cred = '';
if (!empty(Config::get('influxdb.username')) && !empty(Config::get('influxdb.password'))) {
$influxdb_cred = Config::get('influxdb.username') . ':' . Config::get('influxdb.password') . '@';
d_echo('Using authentication for InfluxDB');
}
$influxdb_url = $influxdb_cred . Config::get('influxdb.host') . ':' . Config::get('influxdb.port') . '/' . Config::get('influxdb.db');
d_echo(Config::get('influxdb.transport') . " transport being used");
if (Config::get('influxdb.transport') == 'http') {
$influxdb_conn = 'influxdb';
} elseif (Config::get('influxdb.transport') == 'https') {
$influxdb_conn = 'https+influxdb';
} elseif (Config::get('influxdb.transport') == 'udp') {
$influxdb_conn = 'udp+influxdb';
} else {
echo 'InfluxDB support enabled but no valid transport details provided';
return false;
}
$db = \InfluxDB\Client::fromDSN($influxdb_conn . '://' . $influxdb_url, Config::get('influxdb.timeout'), Config::get('influxdb.verifySSL'));
return($db);
}// end influxdb_connect
function influx_update($device, $measurement, $tags, $fields)
{
global $influxdb;
if ($influxdb !== false) {
$tmp_fields = array();
$tmp_tags['hostname'] = $device['hostname'];
foreach ($tags as $k => $v) {
$v = preg_replace(array('/ /','/,/','/=/'), array('\ ','\,','\='), $v);
if (empty($v)) {
$v = '_blank_';
}
$tmp_tags[$k] = $v;
}
foreach ($fields as $k => $v) {
if ($k == 'time') {
$k = 'rtime';
}
$tmp_fields[$k] = force_influx_data($v);
if ($tmp_fields[$k] === null) {
unset($tmp_fields[$k]);
}
}
d_echo("\nInfluxDB data:\n");
d_echo($measurement);
d_echo($tmp_tags);
d_echo($tmp_fields);
d_echo("\nEND\n");
if (Config::get('noinfluxdb') !== true) {
$points = array(
new InfluxDB\Point(
$measurement,
null, // the measurement value
$tmp_tags,
$tmp_fields // optional additional fields
)
);
try {
$result = $influxdb->writePoints($points);
} catch (Exception $e) {
d_echo("Caught exception: " . $e->getMessage() . PHP_EOL);
d_echo($e->getTrace());
}
} else {
c_echo("[%gInfluxDB Disabled%n]\n");
}//end if
}//end if
}// end influx_update

View File

@ -57,10 +57,6 @@ if (!function_exists('module_selected')) {
require_once $install_dir . '/includes/common.php';
require_once $install_dir . '/includes/dbFacile.php';
require_once $install_dir . '/includes/rrdtool.inc.php';
require_once $install_dir . '/includes/influxdb.inc.php';
require_once $install_dir . '/includes/prometheus.inc.php';
require_once $install_dir . '/includes/opentsdb.inc.php';
require_once $install_dir . '/includes/graphite.inc.php';
require_once $install_dir . '/includes/datastore.inc.php';
require_once $install_dir . '/includes/billing.php';
require_once $install_dir . '/includes/syslog.php';

View File

@ -1,62 +0,0 @@
<?php
/*
* LibreNMS
*
* Copyright (c) 2017 Yacine Benamsili <https://github.com/yac01/librenms.git >
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version. Please see LICENSE.txt at the top level of
* the source code distribution for details.
*/
function opentsdb_update($device, $measurement, $tags, $fields)
{
global $opentsdb;
if (\LibreNMS\Config::get('opentsdb.enable') == true) {
if ($opentsdb != true) {
$opentsdb = fsockopen(\LibreNMS\Config::get('opentsdb.host'), \LibreNMS\Config::get('opentsdb.port'));
}
if ($opentsdb == true) {
d_echo("Connection to OpenTSDB is done\n");
} else {
d_echo("Connection to OpenTSDB has failed\n");
}
$flag = \LibreNMS\Config::get('opentsdb.co');
$timestamp = time();
$tmp_tags = "hostname=".$device['hostname'];
foreach ($tags as $k => $v) {
$v = str_replace(array(' ',',','='), '_', $v);
if (!empty($v)) {
$tmp_tags = $tmp_tags ." ". $k ."=".$v;
}
}
if ($measurement == 'port') {
foreach ($fields as $k => $v) {
$measurement = $k;
if ($flag == true) {
$measurement = $measurement.".".$device['co'];
}
$line = sprintf('put net.port.%s %d %f %s', strtolower($measurement), $timestamp, $v, $tmp_tags);
d_echo("Sending to OPenTSDB: $line\n");
fwrite($opentsdb, $line . "\n"); // send $line into OpenTSDB
}
} else {
if ($flag == true) {
$measurement = $measurement.'.'.$device['co'];
}
foreach ($fields as $k => $v) {
$tmp_tags_key = $tmp_tags ." ". "key" ."=".$k;
$line = sprintf('put net.%s %d %f %s', strtolower($measurement), $timestamp, $v, $tmp_tags_key);
d_echo("Sending to OPenTSDB: $line\n");
fwrite($opentsdb, $line . "\n"); // send $line into OpenTSDB
}
}
}
}

View File

@ -219,6 +219,8 @@ $mapping_status = array(
$rrd_name = array('app', $name, $app_id, 'status');
$rrd_def = new RrdDefinition();
// because this sends different names for rrd and compared to other datastores, disable $fields name checks
$rrd_def->disableNameChecking();
$fields = array();
foreach ($mapping_status as $desc => $id) {

View File

@ -47,83 +47,34 @@ while (isset($lines[$int])) {
$id196, $id197, $id198, $id199, $id231, $id233, $completed, $interrupted, $read_failure,
$unknown_failure, $extended, $short, $conveyance, $selective)=explode(",", $lines[$int]);
if (is_int($id5)) {
$id=null;
}
if (is_int($id10)) {
$id10=null;
}
if (is_int($id173)) {
$id173=null;
}
if (is_int($id177)) {
$id177=null;
}
if (is_int($id183)) {
$id183=null;
}
if (is_int($id184)) {
$id184=null;
}
if (is_int($id187)) {
$id187=null;
}
if (is_int($id188)) {
$id188=null;
}
if (is_int($id190)) {
$id190=null;
}
if (is_int($id194)) {
$id194=null;
}
if (is_int($id196)) {
$id196=null;
}
if (is_int($id197)) {
$id197=null;
}
if (is_int($id198)) {
$id198=null;
}
if (is_int($id199)) {
$id199=null;
}
if (is_int($id231)) {
$id231=null;
}
if (is_int($id233)) {
$id233=null;
}
$rrd_name = array('app', $name, $app_id, $disk);
$fields = array(
'id5'=>$id5,
'id10'=>$id10,
'id173'=>$id173,
'id177'=>$id177,
'id183'=>$id183,
'id184'=>$id184,
'id187'=>$id187,
'id188'=>$id188,
'id190'=>$id190,
'id194'=>$id194,
'id196'=>$id196,
'id197'=>$id197,
'id198'=>$id198,
'id199'=>$id199,
'id231'=>$id231,
'id233'=>$id233,
'completed'=>$completed,
'interrupted'=>$interrupted,
'readfailure'=>$read_failure,
'unknownfail'=>$unknown_failure,
'extended'=>$extended,
'short'=>$short,
'conveyance'=>$conveyance,
'selective'=>$selective
);
$fields = [
'id5' => is_numeric($id5) ? $id5 : null,
'id10' => is_numeric($id10) ? $id10 : null,
'id173' => is_numeric($id173) ? $id173 : null,
'id177' => is_numeric($id177) ? $id177 : null,
'id183' => is_numeric($id183) ? $id183 : null,
'id184' => is_numeric($id184) ? $id184 : null,
'id187' => is_numeric($id187) ? $id187 : null,
'id188' => is_numeric($id188) ? $id188 : null,
'id190' => is_numeric($id190) ? $id190 : null,
'id194' => is_numeric($id194) ? $id194 : null,
'id196' => is_numeric($id196) ? $id196 : null,
'id197' => is_numeric($id197) ? $id197 : null,
'id198' => is_numeric($id198) ? $id198 : null,
'id199' => is_numeric($id199) ? $id199 : null,
'id231' => is_numeric($id231) ? $id231 : null,
'id233' => is_numeric($id233) ? $id233 : null,
'completed' => is_numeric($completed) ? $completed : null,
'interrupted' => is_numeric($interrupted) ? $interrupted : null,
'readfailure' => is_numeric($read_failure) ? $read_failure : null,
'unknownfail' => is_numeric($unknown_failure) ? $unknown_failure : null,
'extended' => is_numeric($extended) ? $extended : null,
'short' => is_numeric($short) ? $short : null,
'conveyance' => is_numeric($conveyance) ? $conveyance : null,
'selective' => is_numeric($selective) ? $selective : null
];
$metrics[$disk] = $fields;
$tags = array('name' => $name, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name);
@ -146,7 +97,7 @@ while (isset($lines[$int])) {
$fields = ['id9' => $id9];
$metrics[$disk]['id9'] = $id9;
$tags = array('name' => $name, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name);
data_update($device, 'app', $tags, $fields);

View File

@ -105,7 +105,7 @@ if ($data) {
if (strstr($stats[$oid], 'No') || strstr($stats[$oid], 'd') || strstr($stats[$oid], 's')) {
$stats[$oid] = '0';
}
$fields[$oid] = $stats[$oid];
$fields[$oid_ds] = $stats[$oid];
}
$tags = compact('af', 'rrd_name', 'rrd_def');

View File

@ -43,7 +43,7 @@ if (!starts_with($device['os'], array('Snom', 'asa'))) {
$fields = array();
foreach ($oids as $oid) {
$rrd_def->addDataset($oid, 'COUNTER', null, 100000000000);
$fields[$oid] = isset($data[$oid]) ? $data[$oid] : 'U';
$fields[substr($oid, 0, 19)] = isset($data[$oid]) ? $data[$oid] : 'U';
}
$tags = compact('rrd_def');

View File

@ -45,7 +45,7 @@ if ($device['os'] != 'Snom') {
$fields = array();
foreach ($oids as $oid) {
$rrd_def->addDataset($oid, 'COUNTER', null, 100000000000);
$fields[$oid] = isset($data[0][$oid]) ? $data[0][$oid] : 'U';
$fields[substr($oid, 0, 19)] = isset($data[0][$oid]) ? $data[0][$oid] : 'U';
}
$tags = compact('rrd_def');

View File

@ -823,16 +823,7 @@ foreach ($ports as $port) {
'OUTMULTICASTPKTS' => $this_port['ifOutMulticastPkts'],
);
if ($tune_port === true) {
rrdtool_tune('port', $rrdfile, $this_port['ifSpeed']);
}
$port_descr_type = $port['port_descr_type'];
$ifName = $port['ifName'];
$ifAlias = $port['ifAlias'];
$tags = compact('ifName', 'ifAlias', 'port_descr_type', 'rrd_name', 'rrd_def');
rrdtool_data_update($device, 'ports', $tags, $fields);
// non rrd stats (will be filtered)
$fields['ifInUcastPkts_rate'] = $port['ifInUcastPkts_rate'];
$fields['ifOutUcastPkts_rate'] = $port['ifOutUcastPkts_rate'];
$fields['ifInErrors_rate'] = $port['ifInErrors_rate'];
@ -844,10 +835,20 @@ foreach ($ports as $port) {
$fields['ifInBits_rate'] = $port['stats']['ifInBits_rate'];
$fields['ifOutBits_rate'] = $port['stats']['ifOutBits_rate'];
prometheus_push($device, 'ports', rrd_array_filter($tags), $fields);
influx_update($device, 'ports', rrd_array_filter($tags), $fields);
graphite_update($device, 'ports|' . $ifName, $tags, $fields);
opentsdb_update($device, 'port', array('ifName' => $this_port['ifName'], 'ifIndex' => getPortRrdName($port_id)), $fields);
if ($tune_port === true) {
Rrd::tune('port', $rrdfile, $this_port['ifSpeed']);
}
$tags = [
'ifName' => $port['ifName'],
'ifAlias' => $port['ifAlias'],
'ifIndex' => $port['ifIndex'],
'port_descr_type' => $port['port_descr_type'],
'rrd_name' => $rrd_name,
'rrd_def' => $rrd_def,
];
app('Datastore')->put($device, 'ports', $tags, $fields);
// End Update IF-MIB
// Update PAgP
if ($this_port['pagpOperationMode'] || $port['pagpOperationMode']) {

View File

@ -49,5 +49,5 @@ if (isset($this_port['cieIfInRuntsErrs'])) {
*/
$ifName = $port['ifName'];
$tags = compact('ifName', 'rrd_name', 'rrd_def');
rrdtool_data_update($device, 'drops', $tags, $rrd_data);
data_update($device, 'drops', $tags, $rrd_data);
}

View File

@ -1,75 +0,0 @@
<?php
use LibreNMS\Config;
/*
* LibreNMS
*
* Copyright (c) 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version. Please see LICENSE.txt at the top level of
* the source code distribution for details.
*/
function prometheus_push($device, $measurement, $tags, $fields)
{
global $prometheus;
if (Config::get('prometheus.enable') === true) {
if ($prometheus !== false) {
try {
$ch = curl_init();
set_curl_proxy($ch);
$vals = "";
$promtags = "/measurement/".$measurement;
foreach ($fields as $k => $v) {
if ($v !== null) {
$vals = $vals . "$k $v\n";
}
}
foreach ($tags as $t => $v) {
if ($v !== null) {
$promtags = $promtags . "/$t/$v";
}
}
$promurl = Config::get('prometheus.url') . '/metrics/job/' . Config::get('prometheus.job') . '/instance/' . $device['hostname'] . $promtags;
$promurl = str_replace(" ", "-", $promurl); // Prometheus doesn't handle tags with spaces in url
d_echo("\nPrometheus data:\n");
d_echo($measurement);
d_echo($tags);
d_echo($fields);
d_echo($vals);
d_echo($promurl);
d_echo("\nEND\n");
curl_setopt($ch, CURLOPT_URL, $promurl);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $vals);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$headers = array();
$headers[] = "Content-Type: test/plain";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_exec($ch);
if (curl_errno($ch)) {
d_echo('Error:' . curl_error($ch));
}
} catch (Exception $e) {
d_echo("Caught exception: " . $e->getMessage() . PHP_EOL);
d_echo($e->getTrace());
}
} else {
c_echo("[%gPrometheus Push Disabled%n]\n");
}//end if
}//end if
}// end prometheus_push

View File

@ -24,71 +24,6 @@
* @author Tony Murray <murraytony@gmail.com>
*/
use LibreNMS\Config;
use LibreNMS\Exceptions\FileExistsException;
use LibreNMS\Proc;
/**
* Opens up a pipe to RRDTool using handles provided
*
* @param bool $dual_process start an additional process that's output should be read after every command
* @return bool the process(s) have been successfully started
*/
function rrdtool_initialize($dual_process = true)
{
global $rrd_sync_process, $rrd_async_process;
$command = Config::get('rrdtool') . ' -';
$descriptor_spec = array(
0 => array('pipe', 'r'), // stdin is a pipe that the child will read from
1 => array('pipe', 'w'), // stdout is a pipe that the child will write to
2 => array('pipe', 'w'), // stderr is a pipe that the child will write to
);
$cwd = Config::get('rrd_dir');
if (!rrdtool_running($rrd_sync_process)) {
$rrd_sync_process = new Proc($command, $descriptor_spec, $cwd);
}
if ($dual_process && !rrdtool_running($rrd_async_process)) {
$rrd_async_process = new Proc($command, $descriptor_spec, $cwd);
$rrd_async_process->setSynchronous(false);
}
return rrdtool_running($rrd_sync_process) && ($dual_process ? rrdtool_running($rrd_async_process) : true);
}
/**
* Checks if the variable is a running rrdtool process
*
* @param $process
* @return bool
*/
function rrdtool_running(&$process)
{
return isset($process) && $process instanceof Proc && $process->isRunning();
}
/**
* Close all open rrdtool processes.
* This should be done before exiting a script that has called rrdtool_initilize()
*/
function rrdtool_close()
{
global $rrd_sync_process, $rrd_async_process;
/** @var Proc $rrd_sync_process */
/** @var Proc $rrd_async_process */
if (rrdtool_running($rrd_sync_process)) {
$rrd_sync_process->close('quit');
}
if (rrdtool_running($rrd_async_process)) {
$rrd_async_process->close('quit');
}
}
/**
* Generates a graph file at $graph_file using $options
* Opens its own rrdtool pipe.
@ -99,123 +34,10 @@ function rrdtool_close()
*/
function rrdtool_graph($graph_file, $options)
{
global $debug, $rrd_sync_process;
/** @var Proc $rrd_sync_process */
if (rrdtool_initialize(false)) {
$cmd = rrdtool_build_command('graph', $graph_file, $options);
$output = implode($rrd_sync_process->sendCommand($cmd));
if ($debug) {
echo "<p>$cmd</p>";
echo "<p>command returned ($output)</p>";
}
return $output;
} else {
return 0;
}
return Rrd::graph($graph_file, $options);
}
/**
* Generates and pipes a command to rrdtool
*
* @internal
* @param string $command create, update, updatev, graph, graphv, dump, restore, fetch, tune, first, last, lastupdate, info, resize, xport, flushcached
* @param string $filename The full patth to the rrd file
* @param string $options rrdtool command options
* @return array the output of stdout and stderr in an array
* @throws FileExistsException thrown when a create command is set to rrdtool < 1.4 and the rrd already exists
* @throws Exception thrown when the rrdtool process(s) cannot be started
*/
function rrdtool($command, $filename, $options)
{
global $debug, $vdebug, $rrd_async_process, $rrd_sync_process;
/** @var Proc $rrd_sync_process */
/** @var Proc $rrd_async_process */
$start_time = microtime(true);
try {
$cmd = rrdtool_build_command($command, $filename, $options);
} catch (FileExistsException $e) {
c_echo('RRD[%g' . $filename . " already exists%n]\n", $debug);
return array(null, null);
}
c_echo("RRD[%g$cmd%n]\n", $debug);
// do not write rrd files, but allow read-only commands
$ro_commands = array('graph', 'graphv', 'dump', 'fetch', 'first', 'last', 'lastupdate', 'info', 'xport');
if (!empty(Config::get('norrd')) && !in_array($command, $ro_commands)) {
c_echo('[%rRRD Disabled%n]', !Config::get('hide_rrd_disabled'));
return array(null, null);
}
// send the command!
if ($command == 'last' && rrdtool_initialize(false)) {
// send this to our synchronous process so output is guaranteed
$output = $rrd_sync_process->sendCommand($cmd);
} elseif (rrdtool_initialize()) {
// don't care about the return of other commands, so send them to the faster async process
$output = $rrd_async_process->sendCommand($cmd);
} else {
throw new Exception('rrdtool could not start');
}
if ($vdebug) {
echo 'RRDtool Output: ';
echo $output[0];
echo $output[1];
}
recordRrdStatistic($command, $start_time);
return $output;
}
/**
* Build a command for rrdtool
* Shortens the filename as needed
* Determines if --daemon and -O should be used
*
* @internal
* @param string $command The base rrdtool command. Usually create, update, last.
* @param string $filename The full path to the rrd file
* @param string $options Options for the command possibly including the rrd definition
* @return string returns a full command ready to be piped to rrdtool
* @throws FileExistsException if rrdtool <1.4.3 and the rrd file exists locally
*/
function rrdtool_build_command($command, $filename, $options)
{
if ($command == 'create') {
// <1.4.3 doesn't support -O, so make sure the file doesn't exist
if (version_compare(Config::get('rrdtool_version', '1.4'), '1.4.3', '<')) {
if (is_file($filename)) {
throw new FileExistsException();
}
} else {
$options .= ' -O';
}
}
// no remote for create < 1.5.5 and tune < 1.5
$rrdtool_version = Config::get('rrdtool_version', '1.4');
if (Config::get('rrdcached') &&
!($command == 'create' && version_compare($rrdtool_version, '1.5.5', '<')) &&
!($command == 'tune' && Config::get('rrdcached') && version_compare($rrdtool_version, '1.5', '<'))
) {
// only relative paths if using rrdcached
$filename = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $filename);
$options = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $options);
return "$command $filename $options --daemon " . Config::get('rrdcached');
}
return "$command $filename $options";
}
/**
* Checks if the rrd file exists on the server
* This will perform a remote check if using rrdcached and rrdtool >= 1.5
@ -225,46 +47,9 @@ function rrdtool_build_command($command, $filename, $options)
*/
function rrdtool_check_rrd_exists($filename)
{
if (Config::get('rrdcached') && version_compare(Config::get('rrdtool_version', '1.4'), '1.5', '>=')) {
$chk = rrdtool('last', $filename, '');
$filename = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $filename);
return !str_contains(implode($chk), "$filename': No such file or directory");
} else {
return is_file($filename);
}
return Rrd::checkRrdExists($filename);
}
/**
* Updates an rrd database at $filename using $options
* Where $options is an array, each entry which is not a number is replaced with "U"
*
* @internal
* @param string $filename
* @param array $data
* @return array|string
*/
function rrdtool_update($filename, $data)
{
$values = array();
// Do some sanitation on the data if passed as an array.
if (is_array($data)) {
$values[] = 'N';
foreach ($data as $v) {
if (!is_numeric($v)) {
$v = 'U';
}
$values[] = $v;
}
$data = implode(':', $values);
return rrdtool('update', $filename, $data);
} else {
return 'Bad options passed to rrdtool_update';
}
} // rrdtool_update
/**
* Escapes strings for RRDtool
@ -312,11 +97,9 @@ function rrdtool_escape($string, $length = null)
*/
function rrd_name($host, $extra, $extension = ".rrd")
{
$filename = safename(is_array($extra) ? implode("-", $extra) : $extra);
return implode("/", array(get_rrd_dir($host), $filename.$extension));
return Rrd::name($host, $extra, $extension);
} // rrd_name
/**
* Generates a path based on the hostname (or IP)
*
@ -329,107 +112,6 @@ function get_rrd_dir($host)
return implode("/", [Config::get('rrd_dir'), $host]);
} // rrd_dir
/**
* Generates a filename for a proxmox cluster rrd
*
* @param $pmxcluster
* @param $vmid
* @param $vmport
* @return string full path to the rrd.
*/
function proxmox_rrd_name($pmxcluster, $vmid, $vmport)
{
$pmxcdir = join('/', [Config::get('rrd_dir'), 'proxmox', safename($pmxcluster)]);
// this is not needed for remote rrdcached
if (!is_dir($pmxcdir)) {
mkdir($pmxcdir, 0775, true);
}
return join('/', array($pmxcdir, safename($vmid.'_netif_'.$vmport.'.rrd')));
}
/**
* Modify an rrd file's max value and trim the peaks as defined by rrdtool
*
* @param string $type only 'port' is supported at this time
* @param string $filename the path to the rrd file
* @param integer $max the new max value
* @return bool
*/
function rrdtool_tune($type, $filename, $max)
{
$fields = array();
if ($type === 'port') {
if ($max < 10000000) {
return false;
}
$max = $max / 8;
$fields = array(
'INOCTETS',
'OUTOCTETS',
'INERRORS',
'OUTERRORS',
'INUCASTPKTS',
'OUTUCASTPKTS',
'INNUCASTPKTS',
'OUTNUCASTPKTS',
'INDISCARDS',
'OUTDISCARDS',
'INUNKNOWNPROTOS',
'INBROADCASTPKTS',
'OUTBROADCASTPKTS',
'INMULTICASTPKTS',
'OUTMULTICASTPKTS'
);
}
if (count($fields) > 0) {
$options = "--maximum " . implode(":$max --maximum ", $fields) . ":$max";
rrdtool('tune', $filename, $options);
}
return true;
} // rrdtool_tune
/**
* rrdtool backend implementation of data_update
*
* Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device device array
* @param string $measurement the name of this measurement (if no rrd_name tag is given, this will be used to name the file)
* @param array $tags tags to pass additional info to rrdtool
* @param array $fields data values to update
*/
function rrdtool_data_update($device, $measurement, $tags, $fields)
{
$rrd_name = $tags['rrd_name'] ?: $measurement;
$step = $tags['rrd_step'] ?: Config::get('rrd.step');
$oldname = $tags['rrd_oldname'];
if (!empty($oldname)) {
rrd_file_rename($device, $oldname, $rrd_name);
}
if (isset($tags['rrd_proxmox_name'])) {
$pmxvars = $tags['rrd_proxmox_name'];
$rrd = proxmox_rrd_name($pmxvars['pmxcluster'], $pmxvars['vmid'], $pmxvars['vmport']);
} else {
$rrd = rrd_name($device['hostname'], $rrd_name);
}
if (isset($tags['rrd_def']) && !rrdtool_check_rrd_exists($rrd)) {
$newdef = "--step $step " . $tags['rrd_def'] . Config::get('rrd_rra');
rrdtool('create', $rrd, $newdef);
}
rrdtool_update($rrd, $fields);
} // rrdtool_data_update
/**
* rename an rrdfile, can only be done on the LibreNMS server hosting the rrd files
*
@ -440,18 +122,5 @@ function rrdtool_data_update($device, $measurement, $tags, $fields)
*/
function rrd_file_rename($device, $oldname, $newname)
{
$oldrrd = rrd_name($device['hostname'], $oldname);
$newrrd = rrd_name($device['hostname'], $newname);
if (is_file($oldrrd) && !is_file($newrrd)) {
if (rename($oldrrd, $newrrd)) {
log_event("Renamed $oldrrd to $newrrd", $device, "poller", 1);
return true;
} else {
log_event("Failed to rename $oldrrd to $newrrd", $device, "poller", 5);
return false;
}
} else {
// we don't need to rename the file
return true;
}
return Rrd::renameFile($device, $oldname, $newname);
} // rrd_file_rename

View File

@ -37,14 +37,4 @@ if (isset($options['g'])) {
$groups = [];
}
if (Config::get('base_url') !== true && \LibreNMS\Config::get('influxdb.enable') === true) {
$influxdb = influxdb_connect();
} else {
$influxdb = false;
}
rrdtool_initialize();
PingCheck::dispatch($groups);
rrdtool_close();

View File

@ -11,6 +11,8 @@
* @copyright (C) 2006 - 2012 Adam Armstrong
*/
use LibreNMS\Data\Store\Datastore;
$init_modules = array();
require __DIR__ . '/includes/init.php';
@ -22,6 +24,7 @@ if (isset($argv[1]) && is_numeric($argv[1])) {
}
set_debug(isset($options['d']));
Datastore::init();
// Wait for schema update, as running during update can break update
if (get_db_schema() < 107) {
@ -29,8 +32,6 @@ if (get_db_schema() < 107) {
exit(1);
}
rrdtool_initialize();
$poller_start = microtime(true);
echo "Starting Polling Session ... \n\n";
@ -164,4 +165,4 @@ if ($poller_time > 300) {
}
echo "\nCompleted in $poller_time sec\n";
rrdtool_close();
Datastore::terminate();

View File

@ -28,6 +28,7 @@
use LibreNMS\Config;
use LibreNMS\Alert\AlertRules;
use LibreNMS\Data\Store\Datastore;
$init_modules = ['polling', 'alerts', 'laravel'];
require __DIR__ . '/includes/init.php';
@ -117,44 +118,11 @@ EOH;
update_os_cache(true); // Force update of OS Cache
}
if (isset($options['r'])) {
Config::set('norrd', true);
}
if (isset($options['f'])) {
Config::set('noinfluxdb', true);
}
if (isset($options['p'])) {
$prometheus = false;
}
if (isset($options['g'])) {
Config::set('nographite', true);
}
if (Config::get('base_url') !== true && Config::get('influxdb.enable') === true) {
$influxdb = influxdb_connect();
} else {
$influxdb = false;
}
if (Config::get('base_url') !== true && Config::get('graphite.enable') === true) {
$graphite = fsockopen(Config::get('graphite.host'), Config::get('graphite.port'));
if ($graphite !== false) {
echo "Connection made to " . Config::get('graphite.host') . " for Graphite support\n";
} else {
echo "Connection to " . Config::get('graphite.host') . " has failed, Graphite support disabled\n";
Config::set('nographite', true);
}
} else {
$graphite = false;
}
// If we've specified modules with -m, use them
$module_override = parse_modules('poller', $options);
set_debug($debug);
rrdtool_initialize();
$datastore = Datastore::init($options);
echo "Starting polling run:\n\n";
$polled_devices = 0;
@ -186,10 +154,6 @@ $poller_end = microtime(true);
$poller_run = ($poller_end - $poller_start);
$poller_time = substr($poller_run, 0, 5);
if ($graphite !== false) {
fclose($graphite);
}
if ($polled_devices) {
dbInsert(array(
'type' => 'poll',
@ -209,8 +173,7 @@ if (!isset($options['q'])) {
}
logfile($string);
rrdtool_close();
Datastore::terminate();
// Remove this for testing
// print_r(get_defined_vars());

View File

@ -4,8 +4,6 @@
$init_modules = array();
require realpath(__DIR__ . '/..') . '/includes/init.php';
rrdtool_initialize();
$options = getopt('h:p:');
$hosts = str_replace('*', '%', mres($options['h']));
@ -30,8 +28,8 @@ foreach (dbFetchRows("SELECT `device_id`,`hostname` FROM `devices` WHERE `hostna
foreach (dbFetchRows("SELECT `port_id`,`ifIndex`,`ifName`,`ifSpeed` FROM `ports` WHERE `ifName` LIKE ? AND `device_id` = ?", array('%'.$ports.'%',$device['device_id'])) as $port) {
echo "Tuning port " . $port['ifName'].".......\n";
$rrdfile = get_port_rrdfile_path($device['hostname'], $port['port_id']);
rrdtool_tune('port', $rrdfile, $port['ifSpeed']);
Rrd::tune('port', $rrdfile, $port['ifSpeed']);
}
}
rrdtool_close();
Rrd::close();

View File

@ -200,4 +200,22 @@ class ConfigTest extends TestCase
$function($config);
$this->config->setValue($config);
}
public function testForget()
{
Config::set('forget.me', 'now');
$this->assertTrue(Config::has('forget.me'));
Config::forget('forget.me');
$this->assertFalse(Config::has('forget.me'));
}
public function testForgetSubtree()
{
Config::set('forget.me.sub', 'yep');
$this->assertTrue(Config::has('forget.me.sub'));
Config::forget('forget.me');
$this->assertFalse(Config::has('forget.me.sub'));
}
}

View File

@ -26,6 +26,7 @@
namespace LibreNMS\Tests;
use LibreNMS\Config;
use LibreNMS\Data\Store\Rrd;
class RrdtoolTest extends TestCase
{
@ -36,25 +37,26 @@ class RrdtoolTest extends TestCase
Config::set('rrdtool_version', '1.4');
Config::set('rrd_dir', '/opt/librenms/rrd');
$cmd = rrdtool_build_command('create', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('create', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('create /opt/librenms/rrd/f o', $cmd);
$cmd = rrdtool_build_command('tune', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('tune', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('tune /opt/librenms/rrd/f o', $cmd);
$cmd = rrdtool_build_command('update', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('update', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('update /opt/librenms/rrd/f o', $cmd);
$this->app->forgetInstance(Rrd::class);
Config::set('rrdtool_version', '1.6');
$cmd = rrdtool_build_command('create', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('create', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('create /opt/librenms/rrd/f o -O', $cmd);
$cmd = rrdtool_build_command('tune', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('tune', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('tune /opt/librenms/rrd/f o', $cmd);
$cmd = rrdtool_build_command('update', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('update', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('update /opt/librenms/rrd/f o', $cmd);
}
@ -64,25 +66,25 @@ class RrdtoolTest extends TestCase
Config::set('rrdtool_version', '1.4');
Config::set('rrd_dir', '/opt/librenms/rrd');
$cmd = rrdtool_build_command('create', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('create', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('create /opt/librenms/rrd/f o', $cmd);
$cmd = rrdtool_build_command('tune', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('tune', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('tune /opt/librenms/rrd/f o', $cmd);
$cmd = rrdtool_build_command('update', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('update', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('update f o --daemon server:42217', $cmd);
$this->app->forgetInstance(Rrd::class);
Config::set('rrdtool_version', '1.6');
$cmd = rrdtool_build_command('create', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('create', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('create f o -O --daemon server:42217', $cmd);
$cmd = rrdtool_build_command('tune', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('tune', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('tune f o --daemon server:42217', $cmd);
$cmd = rrdtool_build_command('update', '/opt/librenms/rrd/f', 'o');
$cmd = $this->buildCommandProxy('update', '/opt/librenms/rrd/f', 'o');
$this->assertEquals('update f o --daemon server:42217', $cmd);
}
@ -93,6 +95,16 @@ class RrdtoolTest extends TestCase
$this->expectException('LibreNMS\Exceptions\FileExistsException');
// use this file, since it is guaranteed to exist
rrdtool_build_command('create', __FILE__, 'o');
$this->buildCommandProxy('create', __FILE__, 'o');
}
private function buildCommandProxy($command, $filename, $options)
{
// todo better tests
$reflection = new \ReflectionClass(Rrd::class);
$method = $reflection->getMethod('buildCommand');
$method->setAccessible(true);
return $method->invokeArgs($this->app->make(Rrd::class), func_get_args());
}
}

View File

@ -0,0 +1,81 @@
<?php
/**
* DatastoreTest.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Tests\Unit\Data;
use LibreNMS\Config;
use LibreNMS\Tests\TestCase;
/**
* @group datastores
*/
class DatastoreTest extends TestCase
{
public function setUp() : void
{
parent::setUp();
Config::forget([
'graphite',
'influxdb',
'opentsdb',
'prometheus',
'rrd',
]);
}
public function testDefaultInitialization()
{
$ds = $this->app->make('Datastore');
$stores = $ds->getStores();
$this->assertCount(1, $stores, 'Incorrect number of default stores enabled');
$this->assertEquals('LibreNMS\Data\Store\Rrd', get_class($stores[0]), 'The default enabled store should be Rrd');
}
public function testInitialization()
{
Config::set('rrd.enable', false);
Config::set('graphite.enable', true);
Config::set('influxdb.enable', true);
Config::set('opentsdb.enable', true);
Config::set('prometheus.enable', true);
$ds = $this->app->make('Datastore');
$stores = $ds->getStores();
$this->assertCount(4, $stores, 'Incorrect number of default stores enabled');
$enabled = array_map('get_class', $stores);
$expected_enabled = [
'LibreNMS\Data\Store\Graphite',
'LibreNMS\Data\Store\InfluxDB',
'LibreNMS\Data\Store\OpenTSDB',
'LibreNMS\Data\Store\Prometheus',
];
$this->assertEquals($expected_enabled, $enabled, 'Expected all non-default stores to be initialized');
}
}

View File

@ -0,0 +1,108 @@
<?php
/**
* GraphiteStoreTest.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Tests\Unit\Data;
use Carbon\Carbon;
use LibreNMS\Data\Store\Graphite;
use LibreNMS\Tests\TestCase;
/**
* @group datastores
*/
class GraphiteStoreTest extends TestCase
{
protected $timestamp = 997464400;
public function setUp() : void
{
parent::setUp();
// fix the date
Carbon::setTestNow(Carbon::createFromTimestamp($this->timestamp));
}
public function tearDown() : void
{
// restore Carbon:now() to normal
Carbon::setTestNow();
parent::tearDown();
}
public function testSocketConnectError()
{
$mockFactory = \Mockery::mock(\Socket\Raw\Factory::class);
$mockFactory->shouldReceive('createClient')
->andThrow('Socket\Raw\Exception', 'Failed to handle connect exception');
new Graphite($mockFactory);
}
public function testSocketWriteError()
{
$mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
$graphite = $this->mockGraphite($mockSocket);
$mockSocket->shouldReceive('write')
->andThrow('Socket\Raw\Exception', 'Did not handle socket exception');
$graphite->put(['hostname' => 'test'], 'fake', ['rrd_name' => 'name'], ['one' => 1]);
}
public function testSimpleWrite()
{
$mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
$graphite = $this->mockGraphite($mockSocket);
$device = ['hostname' => 'testhost'];
$measurement = 'testmeasure';
$tags = ['rrd_name' => 'rrd_name', 'ifName' => 'testifname', 'type' => 'testtype'];
$fields = ['ifIn' => 234234, 'ifOut' => 53453];
$mockSocket->shouldReceive('write')
->with("testhost.testmeasure.rrd_name.ifIn 234234 $this->timestamp\n");
$mockSocket->shouldReceive('write')
->with("testhost.testmeasure.rrd_name.ifOut 53453 $this->timestamp\n");
$graphite->put($device, $measurement, $tags, $fields);
}
/**
* @param $mockSocket
* @return Graphite
*/
private function mockGraphite($mockSocket)
{
$mockFactory = \Mockery::mock(\Socket\Raw\Factory::class);
$mockFactory->shouldReceive('createClient')
->andReturn($mockSocket);
$graphite = new Graphite($mockFactory);
return $graphite;
}
}

View File

@ -0,0 +1,66 @@
<?php
/**
* InfluxStoreTest.php *
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Tests\Unit\Data;
use InfluxDB\Point;
use LibreNMS\Config;
use LibreNMS\Data\Store\InfluxDB;
use LibreNMS\Tests\TestCase;
/**
* @group datastores
*/
class InfluxDBStoreTest extends TestCase
{
public function testBadSettings()
{
Config::set('influxdb.host', '');
Config::set('influxdb.port', 'abc');
$influx = new InfluxDB(InfluxDB::createFromConfig());
\Log::shouldReceive('debug');
\Log::shouldReceive('error')->once()->with('InfluxDB exception: Unable to parse URI: http://:0'); // the important one
$influx->put(['hostname' => 'test'], 'fake', [], ['one' => 1]);
}
public function testSimpleWrite()
{
// Create a mock of the Random Interface
$mock = \Mockery::mock(\InfluxDB\Database::class);
$mock->shouldReceive('exists')->once()->andReturn(true);
$influx = new InfluxDB($mock);
$device = ['hostname' => 'testhost'];
$measurement = 'testmeasure';
$tags = ['ifName' => 'testifname', 'type' => 'testtype'];
$fields = ['ifIn' => 234234, 'ifOut' => 53453];
$expected = [new Point($measurement, null, ['hostname' => $device['hostname']] + $tags, $fields)];
$mock->shouldReceive('writePoints')->withArgs([$expected])->once();
$influx->put($device, $measurement, $tags, $fields);
}
}

View File

@ -0,0 +1,124 @@
<?php
/**
* OpenTSDBStoreTest.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Tests\Unit\Data;
use Carbon\Carbon;
use GuzzleHttp\Client;
use LibreNMS\Data\Store\OpenTSDB;
use LibreNMS\Tests\TestCase;
/**
* @group datastores
*/
class OpenTSDBStoreTest extends TestCase
{
protected $timestamp = 990464400;
public function setUp() : void
{
parent::setUp();
// fix the date
Carbon::setTestNow(Carbon::createFromTimestamp($this->timestamp));
}
public function tearDown() : void
{
// restore Carbon:now() to normal
Carbon::setTestNow();
parent::tearDown();
}
public function testSocketConnectError()
{
$mockFactory = \Mockery::mock(\Socket\Raw\Factory::class);
$mockFactory->shouldReceive('createClient')
->andThrow('Socket\Raw\Exception', 'Failed to handle connect exception');
new OpenTSDB($mockFactory);
}
public function testSocketWriteError()
{
$mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
$opentsdb = $this->mockOpenTSDB($mockSocket);
$mockSocket->shouldReceive('write')
->andThrow('Socket\Raw\Exception', 'Did not handle socket exception');
$opentsdb->put(['hostname' => 'test'], 'fake', [], ['one' => 1]);
}
public function testSimpleWrite()
{
$mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
$opentsdb = $this->mockOpenTSDB($mockSocket);
$device = ['hostname' => 'testhost'];
$measurement = 'testmeasure';
$tags = ['ifName' => 'testifname', 'type' => 'testtype'];
$fields = ['ifIn' => 234234, 'ifOut' => 53453];
$mockSocket->shouldReceive('write')
->with("put net.testmeasure $this->timestamp 234234.000000 hostname=testhost ifName=testifname type=testtype key=ifIn\n");
$mockSocket->shouldReceive('write')
->with("put net.testmeasure $this->timestamp 53453.000000 hostname=testhost ifName=testifname type=testtype key=ifOut\n");
$opentsdb->put($device, $measurement, $tags, $fields);
}
public function testPortWrite()
{
$mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
$opentsdb = $this->mockOpenTSDB($mockSocket);
$device = ['hostname' => 'testhost'];
$measurement = 'ports';
$tags = ['ifName' => 'testifname', 'type' => 'testtype'];
$fields = ['ifIn' => 897238, 'ifOut' => 2342];
$mockSocket->shouldReceive('write')
->with("put net.port.ifin $this->timestamp 897238.000000 hostname=testhost ifName=testifname type=testtype\n");
$mockSocket->shouldReceive('write')
->with("put net.port.ifout $this->timestamp 2342.000000 hostname=testhost ifName=testifname type=testtype\n");
$opentsdb->put($device, $measurement, $tags, $fields);
}
/**
* @param $mockSocket
* @return OpenTSDB
*/
private function mockOpenTSDB($mockSocket)
{
$mockFactory = \Mockery::mock(\Socket\Raw\Factory::class);
$mockFactory->shouldReceive('createClient')
->andReturn($mockSocket);
return new OpenTSDB($mockFactory);
}
}

View File

@ -0,0 +1,105 @@
<?php
/**
* PrometheusStoreTest.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Tests\Unit\Data;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use LibreNMS\Config;
use LibreNMS\Data\Store\Prometheus;
use LibreNMS\Tests\TestCase;
/**
* @group datastores
*/
class PrometheusStoreTest extends TestCase
{
public function setUp() : void
{
parent::setUp();
Config::set('prometheus.enable', true);
Config::set('prometheus.url', 'http://fake:9999');
}
public function testFailWrite()
{
$stack = HandlerStack::create(new MockHandler([
new Response(422, [], 'Bad response'),
new RequestException("Exception thrown", new Request('POST', 'test'))
]));
$client = new Client(['handler' => $stack]);
$prometheus = new Prometheus($client);
\Log::shouldReceive('debug');
\Log::shouldReceive('error')->once()->with("Prometheus Exception: Client error: `POST http://fake:9999/metrics/job/librenms/instance/test/measurement/none` resulted in a `422 Unprocessable Entity` response:\nBad response\n");
\Log::shouldReceive('error')->once()->with('Prometheus Exception: Exception thrown');
$prometheus->put(['hostname' => 'test'], 'none', [], ['one' => 1]);
$prometheus->put(['hostname' => 'test'], 'none', [], ['one' => 1]);
}
public function testSimpleWrite()
{
$stack = HandlerStack::create(new MockHandler([
new Response(200),
]));
$container = [];
$history = Middleware::history($container);
$stack->push($history);
$client = new Client(['handler' => $stack]);
$prometheus = new Prometheus($client);
$device = ['hostname' => 'testhost'];
$measurement = 'testmeasure';
$tags = ['ifName' => 'testifname', 'type' => 'testtype'];
$fields = ['ifIn' => 234234, 'ifOut' => 53453];
\Log::shouldReceive('debug');
\Log::shouldReceive('error')->times(0);
$prometheus->put($device, $measurement, $tags, $fields);
$this->assertCount(1, $container, 'Did not receive the expected number of requests');
/** @var Request $request */
$request = $container[0]['request'];
$this->assertEquals('POST', $request->getMethod());
$this->assertEquals("/metrics/job/librenms/instance/testhost/measurement/testmeasure/ifName/testifname/type/testtype", $request->getUri()->getPath());
$this->assertEquals('fake', $request->getUri()->getHost());
$this->assertEquals(9999, $request->getUri()->getPort());
$this->assertEquals("ifIn 234234\nifOut 53453\n", (string)$request->getBody());
}
}