opensourcemirror
/
LibreNMS
mirror of https://github.com/librenms/librenms.git
1
0
Fork 0
Code Releases Activity

Updated device cache support (#10795) 

* Device Cache
also some additional device related models

* attribs from cache

* replace common attribute functions

* remove legacy cache usage
tidy up some collection manipulation
remove some unused or single use functions

* cleanup some items

* always return a device, to prevent bugs

* clear device cache when testing after each test

* fix double assignment

* Clean up function to take advantage of null object
This commit is contained in:
Tony Murray 2019-11-14 21:56:06 +00:00 committed by Neil Lathwood
parent d1c73b14f3
commit dec9a498ee
20 changed files with 311 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
LibreNMS/Cache/Device.php Normal file
View File

@ -0,0 +1,118 @@
<?php
/**
* Device.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2019 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Cache;
class Device
{
private $devices = [];
private $primary;
/**
* Gets the current primary device.
*
* @return \App\Models\Device
*/
public function getPrimary() : \App\Models\Device
{
return self::get($this->primary);
}
/**
* Set the primary device.
* This will be fetched by getPrimary()
*
* @param int $device_id
*/
public function setPrimary($device_id)
{
$this->primary = $device_id;
}
/**
* Get a device by device_id
*
* @param int $device_id
* @return \App\Models\Device
*/
public function get($device_id) : \App\Models\Device
{
if (!array_key_exists($device_id, $this->devices)) {
return self::load($device_id);
}
return $this->devices[$device_id] ?: new \App\Models\Device;
}
/**
* Get a device by hostname
*
* @param string $hostname
* @return \App\Models\Device
*/
public function getByHostname($hostname) : \App\Models\Device
{
$device_id = collect($this->devices)->pluck('device_id', 'hostname')->get($hostname);
if (!$device_id) {
return $this->load($hostname, 'hostname');
}
return $this->devices[$device_id] ?: new \App\Models\Device;
}
/**
* Ignore cache and load the device fresh from the database
*
* @param int $device_id
* @return \App\Models\Device
*/
public function refresh($device_id) : \App\Models\Device
{
unset($this->devices[$device_id]);
return self::get($device_id);
}
/**
* Flush the cache
*/
public function flush()
{
$this->devices = [];
}
private function load($value, $field = 'device_id')
{
$device = \App\Models\Device::query()->where($field, $value)->first();
if (!$device) {
return new \App\Models\Device;
}
$device->loadOs();
$this->devices[$device->device_id] = $device;
return $device;
}
}

36
app/Facades/DeviceCache.php Normal file
View File

@ -0,0 +1,36 @@
<?php
/**
* DeviceCache.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2019 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace App\Facades;
use Illuminate\Support\Facades\Facade;
class DeviceCache extends Facade
{
protected static function getFacadeAccessor()
{
return 'device-cache';
}
}

50
app/Models/Device.php
View File

@ -7,6 +7,7 @@ use Fico7489\Laravel\Pivot\Traits\PivotEventTrait;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\ModelNotFoundException;
use Illuminate\Database\Query\JoinClause;
use Illuminate\Support\Collection;
use Illuminate\Support\Str;
use LibreNMS\Exceptions\InvalidIpException;
use LibreNMS\Util\IP;
@ -334,6 +335,45 @@ class Device extends BaseModel
$this->save();
}
public function getAttrib($name)
{
return $this->attribs->pluck('attrib_value', 'attrib_type')->get($name);
}
public function setAttrib($name, $value)
{
$attrib = $this->attribs->first(function ($item) use ($name) {
return $item->attrib_type === $name;
});
if (!$attrib) {
$attrib = new DeviceAttrib(['attrib_type' => $name]);
$this->attribs->push($attrib);
}
$attrib->attrib_value = $value;
return (bool)$this->attribs()->save($attrib);
}
public function forgetAttrib($name)
{
$attrib_index = $this->attribs->search(function ($attrib) use ($name) {
return $attrib->attrib_type === $name;
});
if ($attrib_index !== false) {
$this->attribs->forget($attrib_index);
return (bool)$this->attribs->get($attrib_index)->delete();
}
return false;
}
public function getAttribs()
{
return $this->attribs->pluck('attrib_value', 'attrib_type')->toArray();
}
// ---- Accessors/Mutators ----
public function getIconAttribute($icon)
@ -445,6 +485,11 @@ class Device extends BaseModel
return $this->hasMany('App\Models\Alert', 'device_id');
}
public function attribs()
{
return $this->hasMany('App\Models\DeviceAttrib', 'device_id');
}
public function alertSchedules()
{
return $this->morphToMany('App\Models\AlertSchedule', 'alert_schedulable', 'alert_schedulables', 'schedule_id', 'schedule_id');
@ -606,6 +651,11 @@ class Device extends BaseModel
return $this->belongsToMany('App\Models\User', 'devices_perms', 'device_id', 'user_id');
}
public function vrfLites()
{
return $this->hasMany('App\Models\VrfLite', 'device_id');
}
public function vrfs()
{
return $this->hasMany('App\Models\Vrf', 'device_id');

34
app/Models/DeviceAttrib.php Normal file
View File

@ -0,0 +1,34 @@
<?php
/**
* DeviceAttrib.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2019 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace App\Models;
class DeviceAttrib extends DeviceRelatedModel
{
protected $table = 'devices_attribs';
protected $primaryKey = 'attrib_id';
public $timestamps = false;
protected $fillable = ['attrib_type', 'attrib_value'];
}

33
app/Models/VrfLite.php Normal file
View File

@ -0,0 +1,33 @@
<?php
/**
* VrfLite.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2019 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace App\Models;
class VrfLite extends DeviceRelatedModel
{
protected $table = 'vrf_lite_cisco';
protected $primaryKey = 'vrf_lite_cisco_id';
public $timestamps = false;
}

4
app/Providers/AppServiceProvider.php
View File

@ -6,7 +6,6 @@ use Illuminate\Database\Eloquent\Relations\Relation;
use Illuminate\Support\Facades\Blade;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\ServiceProvider;
use Illuminate\Validation\Rule;
use LibreNMS\Config;
use LibreNMS\Permissions;
use LibreNMS\Util\IP;
@ -28,6 +27,9 @@ class AppServiceProvider extends ServiceProvider
$this->app->singleton('permissions', function ($app) {
return new Permissions();
});
$this->app->singleton('device-cache', function ($app) {
return new \LibreNMS\Cache\Device();
});
}
/**

1
config/app.php
View File

@ -240,6 +240,7 @@ return [
// LibreNMS
'Permissions' => \App\Facades\Permissions::class,
'DeviceCache' => \App\Facades\DeviceCache::class,
],
];

1
discovery.php
View File

@ -117,6 +117,7 @@ if (!empty(\LibreNMS\Config::get('distributed_poller_group'))) {
global $device;
foreach (dbFetch("SELECT * FROM `devices` WHERE disabled = 0 AND snmp_disable = 0 $where ORDER BY device_id DESC", $sqlparams) as $device) {
DeviceCache::setPrimary($device['device_id']);
$discovered_devices += (int)discover_device($device, $module_override);
}

129
includes/common.php
View File

@ -16,6 +16,7 @@
* the source code distribution for details.
*/
use App\Models\Device;
use LibreNMS\Config;
use LibreNMS\Exceptions\InvalidIpException;
use LibreNMS\Util\Git;
@ -239,23 +240,6 @@ function get_port_by_ifIndex($device_id, $ifIndex)
return dbFetchRow("SELECT * FROM `ports` WHERE `device_id` = ? AND `ifIndex` = ?", array($device_id, $ifIndex));
}
function get_all_devices()
{
global $cache;
$devices = array();
// FIXME needs access control checks!
// FIXME respect $type (server, network, etc) -- needs an array fill in topnav.
if (isset($cache['devices']['hostname'])) {
$devices = array_keys($cache['devices']);
} else {
$devices = dbFetchRows("SELECT * FROM `devices` ORDER BY hostname");
}
return $devices;
}
function table_from_entity_type($type)
{
// Fuck you, english pluralisation.
@ -347,10 +331,9 @@ function ifclass($ifOperStatus, $ifAdminStatus)
return \LibreNMS\Util\Url::portLinkDisplayClass((object) ['ifOperStatus' => $ifOperStatus, 'ifAdminStatus' => $ifAdminStatus]);
}
function device_by_name($name, $refresh = 0)
function device_by_name($name)
{
// FIXME - cache name > id too.
return device_by_id_cache(getidbyname($name), $refresh);
return device_by_id_cache(getidbyname($name));
}
@ -365,29 +348,15 @@ function accesspoint_by_id($ap_id, $refresh = '0')
function device_by_id_cache($device_id, $refresh = false)
{
global $cache;
$model = $refresh ? DeviceCache::refresh($device_id) : DeviceCache::get($device_id);
if (!$refresh && isset($cache['devices']['id'][$device_id]) && is_array($cache['devices']['id'][$device_id])) {
$device = $cache['devices']['id'][$device_id];
} else {
$device = dbFetchRow("SELECT `devices`.*, `location`, `lat`, `lng` FROM `devices` LEFT JOIN locations ON `devices`.location_id=`locations`.`id` WHERE `device_id` = ?", [$device_id]);
$device['attribs'] = get_dev_attribs($device['device_id']);
load_os($device);
$device = $model->toArray();
$device['location'] = $model->location->location;
$device['lat'] = $model->location->lat;
$device['lng'] = $model->location->lng;
$device['attribs'] = $model->getAttribs();
$device['vrf_lite_cisco'] = $model->vrfLites->keyBy('context_name')->toArray();
//order vrf_lite_cisco with context, this will help to get the vrf_name and instance_name all the time
$vrfs_lite_cisco = dbFetchRows("SELECT * FROM `vrf_lite_cisco` WHERE `device_id` = ?", array($device_id));
if (!empty($vrfs_lite_cisco)) {
$device['vrf_lite_cisco'] = array();
foreach ($vrfs_lite_cisco as $vrf) {
$device['vrf_lite_cisco'][$vrf['context_name']] = $vrf;
}
}
if (!empty($device['ip'])) {
$device['ip'] = inet6_ntop($device['ip']);
}
$cache['devices']['id'][$device_id] = $device;
}
return $device;
}
@ -419,17 +388,9 @@ function getifhost($id)
return dbFetchCell("SELECT `device_id` from `ports` WHERE `port_id` = ?", array($id));
}
function gethostbyid($id)
function gethostbyid($device_id)
{
global $cache;
if (isset($cache['devices']['id'][$id]['hostname'])) {
$hostname = $cache['devices']['id'][$id]['hostname'];
} else {
$hostname = dbFetchCell("SELECT `hostname` FROM `devices` WHERE `device_id` = ?", array($id));
}
return $hostname;
return DeviceCache::get($device_id)->hostname;
}
function strgen($length = 16)
@ -470,28 +431,7 @@ function getifdescrbyid($id)
function getidbyname($hostname)
{
global $cache;
if (isset($cache['devices']['hostname'][$hostname])) {
$id = $cache['devices']['hostname'][$hostname];
} else {
$id = dbFetchCell("SELECT `device_id` FROM `devices` WHERE `hostname` = ?", array($hostname));
}
return $id;
}
function gethostosbyid($id)
{
global $cache;
if (isset($cache['devices']['id'][$id]['os'])) {
$os = $cache['devices']['id'][$id]['os'];
} else {
$os = dbFetchCell("SELECT `os` FROM `devices` WHERE `device_id` = ?", array($id));
}
return $os;
return DeviceCache::getByHostname($hostname)->device_id;
}
function safename($name)
@ -516,21 +456,12 @@ function zeropad($num, $length = 2)
function set_dev_attrib($device, $attrib_type, $attrib_value)
{
if (dbFetchCell("SELECT COUNT(*) FROM devices_attribs WHERE `device_id` = ? AND `attrib_type` = ?", array($device['device_id'],$attrib_type))) {
$return = dbUpdate(array('attrib_value' => $attrib_value), 'devices_attribs', 'device_id=? and attrib_type=?', array($device['device_id'], $attrib_type));
} else {
$return = dbInsert(array('device_id' => $device['device_id'], 'attrib_type' => $attrib_type, 'attrib_value' => $attrib_value), 'devices_attribs');
}
return $return;
return DeviceCache::get($device['device_id'])->setAttrib($attrib_type, $attrib_value);
}
function get_dev_attribs($device)
function get_dev_attribs($device_id)
{
$attribs = array();
foreach (dbFetchRows("SELECT * FROM devices_attribs WHERE `device_id` = ?", array($device)) as $entry) {
$attribs[$entry['attrib_type']] = $entry['attrib_value'];
}
return $attribs;
return DeviceCache::get($device_id)->getAttribs();
}
function get_dev_entity_state($device)
@ -543,36 +474,14 @@ function get_dev_entity_state($device)
return $state;
}
function get_dev_attrib($device, $attrib_type, $attrib_value = '')
function get_dev_attrib($device, $attrib_type)
{
$sql = '';
$params = array($device['device_id'], $attrib_type);
if (!empty($attrib_value)) {
$sql = " AND `attrib_value`=?";
array_push($params, $attrib_value);
}
if ($row = dbFetchRow("SELECT attrib_value FROM devices_attribs WHERE `device_id` = ? AND `attrib_type` = ? $sql", $params)) {
return $row['attrib_value'];
} else {
return null;
}
}
function is_dev_attrib_enabled($device, $attrib, $default = true)
{
$val = get_dev_attrib($device, $attrib);
if ($val != null) {
// attribute is set
return ($val != 0);
} else {
// attribute not set
return $default;
}
return DeviceCache::get($device['device_id'])->getAttrib($attrib_type);
}
function del_dev_attrib($device, $attrib_type)
{
return dbDelete('devices_attribs', "`device_id` = ? AND `attrib_type` = ?", array($device['device_id'], $attrib_type));
return DeviceCache::get($device['device_id'])->forgetAttrib($attrib_type);
}
function formatRates($value, $round = '2', $sf = '3')

3
includes/discovery/functions.inc.php
View File

@ -126,9 +126,8 @@ function discover_device(&$device, $force_module = false)
$valid = array();
// Reset $valid array
$attribs = get_dev_attribs($device['device_id']);
$attribs = DeviceCache::getPrimary()->getAttribs();
$device['attribs'] = $attribs;
$device['snmp_max_repeaters'] = $attribs['snmp_max_repeaters'];
$device_start = microtime(true);
// Start counting device poll time

4
includes/functions.php
View File

@ -2470,8 +2470,8 @@ function get_db_schema()
function get_device_oid_limit($device)
{
// device takes priority
if ($device['snmp_max_oid'] > 0) {
return $device['snmp_max_oid'];
if ($device['attribs']['snmp_max_oid'] > 0) {
return $device['attribs']['snmp_max_oid'];
}
// then os

7
includes/html/object-cache.inc.php
View File

@ -5,13 +5,6 @@
// It would be interesting to know where this is used. It probably should have its own API.
use LibreNMS\ObjectCache;
foreach (dbFetchRows('SELECT * FROM `devices` ORDER BY `hostname`') as $device) {
$cache['devices']['hostname'][$device['hostname']] = $device['device_id'];
$cache['devices']['id'][$device['device_id']] = $device;
$cache['device_types'][$device['type']]++;
}
$devices = new ObjectCache('devices');
$ports = new ObjectCache('ports');
$services = new ObjectCache('services');

3
includes/html/pages/device.inc.php
View File

@ -17,6 +17,7 @@ if (device_permitted($vars['device']) || $permitted_by_port) {
}
$select = array($tab => 'class="active"');
DeviceCache::setPrimary($vars['device']);
$device = device_by_id_cache($vars['device']);
$attribs = get_dev_attribs($device['device_id']);
$device['attribs'] = $attribs;
@ -381,7 +382,7 @@ if (device_permitted($vars['device']) || $permitted_by_port) {
}
if ($device_config_file) {
if (!get_dev_attrib($device, 'override_Oxidized_disable', 'true')) {
if (!get_dev_attrib($device, 'override_Oxidized_disable') === 'true') {
echo '<li class="'.$select['showconfig'].'">
<a href="'.generate_device_url($device, array('tab' => 'showconfig')).'">
<i class="fa fa-align-justify fa-lg icon-theme" aria-hidden="true"></i> Config

2
includes/html/pages/device/edit/apps.inc.php
View File

@ -28,7 +28,7 @@ foreach ($applications as $app) {
if (isset($enabled_apps[$app])) {
$modifiers = ' checked';
if ($enabled_apps[$app]
&& is_dev_attrib_enabled($device, "poll_applications", Config::getOsSetting($device['os'], "poller_modules.applications"))
&& (get_dev_attrib($device, 'poll_applications') || Config::getOsSetting($device['os'], "poller_modules.applications"))
) {
$app_text .= '<span class="text-success"> (Discovered)</span>';
$modifiers .= ' disabled';

5
includes/polling/functions.inc.php
View File

@ -232,14 +232,11 @@ function poll_device($device, $force_module = false)
$device_start = microtime(true);
$attribs = get_dev_attribs($device['device_id']);
$attribs = DeviceCache::getPrimary()->getAttribs();
$device['attribs'] = $attribs;
load_os($device);
$device['snmp_max_repeaters'] = $attribs['snmp_max_repeaters'];
$device['snmp_max_oid'] = $attribs['snmp_max_oid'];
unset($array);
// Start counting device poll time

2
includes/snmp.inc.php
View File

@ -1381,7 +1381,7 @@ function snmpwalk_array_num($device, $oid, $indexes = 1)
*/
function get_device_max_repeaters($device)
{
return $device['snmp_max_repeaters'] ?:
return $device['attribs']['snmp_max_repeaters'] ??
Config::getOsSetting($device['os'], 'snmp.max_repeaters', false);
}

1
poller.php
View File

@ -164,6 +164,7 @@ if (!isset($query)) {
}
foreach (dbFetch($query) as $device) {
DeviceCache::setPrimary($device['device_id']);
if ($device['os_group'] == 'cisco') {
$device['vrf_lite_cisco'] = dbFetchRows("SELECT * FROM `vrf_lite_cisco` WHERE `device_id` = " . $device['device_id']);
} else {

6
scripts/collect-port-polling.php
View File

@ -30,6 +30,8 @@ if (isset($options['help'])) {
exit(0);
}
$where = '';
$params = [];
if (isset($options['h'])) {
if (is_numeric($options['h'])) {
$where = "AND `device_id` = ?";
@ -43,10 +45,8 @@ if (isset($options['h'])) {
$where = "AND `hostname` LIKE ?";
$params = array(str_replace('*', '%', mres($options['h'])));
}
$devices = dbFetch("SELECT * FROM `devices` WHERE status = 1 AND disabled = 0 $where ORDER BY `hostname` ASC", $params);
} else {
$devices = get_all_devices();
}
$devices = dbFetchRows("SELECT * FROM `devices` WHERE status = 1 AND disabled = 0 $where ORDER BY `hostname` ASC", $params);
if (isset($options['e'])) {
if (!is_numeric($options['e']) || $options['e'] < 0) {

1
tests/OSDiscoveryTest.php
View File

@ -130,6 +130,7 @@ class OSDiscoveryTest extends TestCase
'community' => $community,
'os' => 'generic',
'os_group' => '',
'attribs' => [],
];
}

3
tests/OSModulesTest.php
View File

@ -25,6 +25,7 @@
namespace LibreNMS\Tests;
use DeviceCache;
use LibreNMS\Config;
use LibreNMS\Exceptions\FileNotFoundException;
use LibreNMS\Exceptions\InvalidModuleException;
@ -132,6 +133,8 @@ class OSModulesTest extends DBTestCase
. "\nOS $os: Polled $module data does not match that found in $filename"
);
}
DeviceCache::flush(); // clear cached devices
}
public function dumpedDataProvider()