MAC Vendor OUI use scheduler (#15187)

* MAC Vendor OUI use scheduler
Add command to update `lnms maintenance:fetch-ouis`
Show vendor column in tables if mac_oui.enabled is set to true
Improve scheduler validation handle non-standard install directories and systems without systemd
Add index to table to improve speed and improve mac->vendor lookup speed
Scheduled weekly with random wait to prevent stampeding herd issues for upstream
drop oui update from daily

* MAC Vendor OUI use scheduler
Add command to update `lnms maintenance:fetch-ouis`
Show vendor column in tables if mac_oui.enabled is set to true

* Lint fixes and better prefix detection

* update schema file
This commit is contained in:
Tony Murray 2023-08-03 19:29:30 -05:00 committed by GitHub
parent 5a56e9081e
commit 12f8bb2040
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 273 additions and 135 deletions

View File

@ -26,6 +26,7 @@
namespace LibreNMS\Util;
use App\Models\Device;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use LibreNMS\Config;
@ -151,20 +152,29 @@ class Rewrite
* Extract the OUI and match it against database values
*
* @param string $mac
* @return string|null
* @return string
*/
public static function readableOUI($mac)
public static function readableOUI($mac): string
{
$oui = substr($mac, 0, 6);
$result = DB::table('vendor_ouis')->where('oui', $oui)->value('vendor');
$results = DB::table('vendor_ouis')
->where('oui', 'like', "$oui%") // possible matches
->orderBy('oui', 'desc') // so we can check longer ones first if we have them
->pluck('vendor', 'oui');
if ($result === 'IEEE Registration Authority') {
// Then we may have a shorter prefix, so let's try them one after the other, ordered by probability
$result = DB::table('vendor_ouis')->whereIn('oui', [substr($mac, 0, 9), substr($mac, 0, 7)])->value('vendor');
if (count($results) == 1) {
return Arr::first($results);
}
return $result ?: '';
// Then we may have a shorter prefix, so let's try them one after the other
foreach ($results as $oui => $vendor) {
if (str_starts_with($mac, $oui)) {
return $vendor;
}
}
return '';
}
/**

View File

@ -23,6 +23,8 @@
namespace LibreNMS\Validations;
use Illuminate\Support\Facades\Cache;
use LibreNMS\Config;
use LibreNMS\ValidationResult;
use LibreNMS\Validator;
class Scheduler extends BaseValidation
@ -36,8 +38,47 @@ class Scheduler extends BaseValidation
public function validate(Validator $validator): void
{
if (! Cache::has('scheduler_working')) {
$validator->fail('Scheduler is not running',
"cp /opt/librenms/dist/librenms-scheduler.service /opt/librenms/dist/librenms-scheduler.timer /etc/systemd/system/\nsystemctl enable librenms-scheduler.timer\nsystemctl start librenms-scheduler.timer");
$commands = $this->generateCommands($validator);
$validator->result(ValidationResult::fail('Scheduler is not running')->setFix($commands));
}
}
/**
* @param Validator $validator
* @return array
*/
private function generateCommands(Validator $validator): array
{
$commands = [];
$systemctl_bin = Config::locateBinary('systemctl');
$base_dir = rtrim($validator->getBaseDir(), '/');
if (is_executable($systemctl_bin)) {
// systemd exists
if ($base_dir === '/opt/librenms') {
// standard install dir
$commands[] = 'sudo cp /opt/librenms/dist/librenms-scheduler.service /opt/librenms/dist/librenms-scheduler.timer /etc/systemd/system/';
} else {
// non-standard install dir
$commands[] = "sudo sh -c 'sed \"s#/opt/librenms#$base_dir#\" $base_dir/dist/librenms-scheduler.service > /etc/systemd/system/librenms-scheduler.service'";
$commands[] = "sudo sh -c 'sed \"s#/opt/librenms#$base_dir#\" $base_dir/dist/librenms-scheduler.timer > /etc/systemd/system/librenms-scheduler.timer'";
}
$commands[] = 'sudo systemctl enable librenms-scheduler.timer';
$commands[] = 'sudo systemctl start librenms-scheduler.timer';
return $commands;
}
// non-systemd use cron
if ($base_dir === '/opt/librenms') {
$commands[] = 'sudo cp /opt/librenms/dist/librenms-scheduler.cron /etc/cron.d/';
return $commands;
}
// non-standard install dir
$commands[] = "sudo sh -c 'sed \"s#/opt/librenms#$base_dir#\" $base_dir/dist/librenms-scheduler.cron > /etc/cron.d/librenms-scheduler.cron'";
return $commands;
}
}

View File

@ -0,0 +1,136 @@
<?php
namespace App\Console\Commands;
use App\Console\LnmsCommand;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use LibreNMS\Config;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class MaintenanceFetchOuis extends LnmsCommand
{
/**
* The name of the console command.
*
* @var string
*/
protected $name = 'maintenance:fetch-ouis';
protected string $mac_oui_url = 'https://www.wireshark.org/download/automated/data/manuf';
protected int $min_refresh_days = 6;
protected int $max_wait_seconds = 900;
protected int $upsert_chunk_size = 1000;
public function __construct()
{
parent::__construct();
$this->addOption('force', null, InputOption::VALUE_NONE);
$this->addOption('wait', null, InputOption::VALUE_NONE);
}
/**
* Execute the console command.
*/
public function handle(): int
{
$force = $this->option('force');
if (Config::get('mac_oui.enabled') !== true && ! $force) {
$this->line(trans('commands.maintenance:fetch-ouis.disabled', ['setting' => 'mac_oui.enabled']));
if (! $this->confirm(trans('commands.maintenance:fetch-ouis.enable_question'))) {
return 0;
}
Config::persist('mac_oui.enabled', true);
}
// We want to refresh after at least 6 days
$lock = Cache::lock('vendor_oui_db_refresh', 86400 * $this->min_refresh_days);
if (! $lock->get() && ! $force) {
$this->warn(trans('commands.maintenance:fetch-ouis.recently_fetched'));
return 0;
}
// wait for 0-15 minutes to prevent stampeding herd
if ($this->option('wait')) {
$seconds = rand(1, $this->max_wait_seconds);
$minutes = (int) round($seconds / 60);
$this->info(trans_choice('commands.maintenance:fetch-ouis.waiting', $minutes, ['minutes' => $minutes]));
sleep($seconds);
}
$this->line(trans('commands.maintenance:fetch-ouis.starting'));
try {
$this->line(' -> ' . trans('commands.maintenance:fetch-ouis.downloading') . ' ...');
$csv_data = \LibreNMS\Util\Http::client()->get($this->mac_oui_url)->body();
// convert the csv into an array to be consumed by upsert
$this->line(' -> ' . trans('commands.maintenance:fetch-ouis.processing') . ' ...');
$ouis = $this->buildOuiList($csv_data);
$this->line(' -> ' . trans('commands.maintenance:fetch-ouis.saving') . ' ...');
$count = 0;
foreach (array_chunk($ouis, $this->upsert_chunk_size) as $oui_chunk) {
$count += DB::table('vendor_ouis')->upsert($oui_chunk, 'oui');
}
$this->info(trans_choice('commands.maintenance:fetch-ouis.success', $count, ['count' => $count]));
return 0;
} catch (\Exception|\ErrorException $e) {
$this->error(trans('commands.maintenance:fetch-ouis.error'));
$this->error('Exception: ' . get_class($e));
$this->error($e);
$lock->release(); // We did not succeed, so we'll try again next time
return 1;
}
}
private function buildOuiList(string $csv_data): array
{
$ouis = [];
foreach (explode("\n", rtrim($csv_data)) as $csv_line) {
// skip comments
if (str_starts_with($csv_line, '#')) {
continue;
}
[$oui, $vendor] = str_getcsv($csv_line, "\t");
$oui = strtolower(str_replace(':', '', $oui)); // normalize oui
$prefix_index = strpos($oui, '/');
// check for non-/24 oui
if ($prefix_index !== false) {
// find prefix length
$prefix_length = (int) substr($oui, $prefix_index + 1);
// 4 bits per character: /28 = 7 /36 = 9
$substring_length = (int) floor($prefix_length / 4);
$oui = substr($oui, 0, $substring_length);
}
// Add to the list of vendor ids
$ouis[] = [
'vendor' => $vendor,
'oui' => $oui,
];
if ($this->verbosity == OutputInterface::VERBOSITY_DEBUG) {
$this->line(trans('commands.maintenance:fetch-ouis.vendor_update', ['vendor' => $vendor, 'oui' => $oui]));
}
}
return $ouis;
}
}

View File

@ -2,9 +2,11 @@
namespace App\Console;
use App\Console\Commands\MaintenanceFetchOuis;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use Illuminate\Support\Facades\Cache;
use LibreNMS\Config;
use LibreNMS\Util\Debug;
use LibreNMS\Util\Version;
@ -19,6 +21,7 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule): void
{
$this->scheduleMarkWorking($schedule);
$this->scheduleMaintenance($schedule); // should be after all others
}
/**
@ -73,4 +76,17 @@ class Kernel extends ConsoleKernel
Cache::put('scheduler_working', now(), now()->addMinutes(6));
})->everyFiveMinutes();
}
/**
* Schedule maintenance tasks
*/
private function scheduleMaintenance(Schedule $schedule): void
{
$maintenance_log_file = Config::get('log_dir') . '/maintenance.log';
$schedule->command(MaintenanceFetchOuis::class, ['--wait'])
->weeklyOn(0, '1:00')
->onOneServer()
->appendOutputTo($maintenance_log_file);
}
}

View File

@ -358,15 +358,6 @@ if ($options['f'] === 'peeringdb') {
}
}
if ($options['f'] === 'mac_oui') {
$lock = Cache::lock('vendor_oui_db', 86000);
if ($lock->get()) {
$res = mac_oui_to_database();
$lock->release();
exit($res);
}
}
if ($options['f'] === 'refresh_os_cache') {
echo 'Clearing OS cache' . PHP_EOL;
if (is_file(Config::get('install_dir') . '/cache/os_defs.cache')) {

View File

@ -341,7 +341,6 @@ main () {
# and clean up the db.
status_run 'Updating SQL-Schema' './lnms migrate --force --no-interaction --isolated'
status_run 'Cleaning up DB' "'$DAILY_SCRIPT' cleanup"
status_run 'Updating Mac OUI data' "$DAILY_SCRIPT mac_oui"
;;
post-pull)
# re-check dependencies after pull with the new code
@ -371,7 +370,6 @@ main () {
status_run 'Cleaning up DB' "$DAILY_SCRIPT cleanup"
status_run 'Fetching notifications' "$DAILY_SCRIPT notifications"
status_run 'Caching PeeringDB data' "$DAILY_SCRIPT peeringdb"
status_run 'Caching Mac OUI data' "$DAILY_SCRIPT mac_oui"
;;
cleanup)
# Cleanups
@ -406,10 +404,6 @@ main () {
peeringdb)
options=("peeringdb")
call_daily_php "${options[@]}"
;;
mac_oui)
options=("mac_oui")
call_daily_php "${options[@]}"
esac
fi
}

View File

@ -0,0 +1,30 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('vendor_ouis', function (Blueprint $table) {
$table->string('oui', 12)->change();
$table->unique(['oui']);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('vendor_ouis', function (Blueprint $table) {
$table->dropUnique('vendor_ouis_oui_unique');
$table->string('oui')->change();
});
}
};

1
dist/librenms-scheduler.cron vendored Normal file
View File

@ -0,0 +1 @@
* * * * * php /opt/librenms/artisan schedule:run --no-ansi --no-interaction > /dev/null 2>&1

View File

@ -6,6 +6,6 @@ Type=oneshot
StandardOutput=null
StandardError=null
WorkingDirectory=/opt/librenms/
ExecStart=/usr/bin/env php artisan schedule:run
ExecStart=/usr/bin/env php artisan schedule:run --no-ansi --no-interaction
User=librenms
Group=librenms

View File

@ -1163,83 +1163,6 @@ function q_bridge_bits2indices($hex_data)
return $indices;
}
/**
* Function to generate Mac OUI Cache
*/
function mac_oui_to_database()
{
// Refresh timer
$mac_oui_refresh_int_min = 86400 * rand(7, 11); // 7 days + a random number between 0 and 4 days
$lock = Cache::lock('vendor_oui_db_refresh', $mac_oui_refresh_int_min); // We want to refresh after at least $mac_oui_refresh_int_min
if (Config::get('mac_oui.enabled') !== true) {
echo 'Mac OUI integration disabled' . PHP_EOL;
return 0;
}
if ($lock->get()) {
echo 'Storing Mac OUI in the database' . PHP_EOL;
try {
$mac_oui_url = 'https://www.wireshark.org/download/automated/data/manuf';
//$mac_oui_url = 'https://gitlab.com/wireshark/wireshark/-/raw/master/manuf';
//$mac_oui_url_mirror = 'https://raw.githubusercontent.com/wireshark/wireshark/master/manuf';
echo ' -> Downloading ...' . PHP_EOL;
$get = \LibreNMS\Util\Http::client()->get($mac_oui_url);
echo ' -> Processing CSV ...' . PHP_EOL;
$csv_data = $get->body();
// Process each line of the CSV data
foreach (explode("\n", $csv_data) as $csv_line) {
unset($oui);
$entry = str_getcsv($csv_line, "\t");
$length = strlen($entry[0]);
$prefix = strtolower(str_replace(':', '', $entry[0]));
$vendor = $entry[1];
if (is_array($entry) && count($entry) >= 2 && $length == 8) {
// We have a standard OUI xx:xx:xx
$oui = $prefix;
} elseif (is_array($entry) && count($entry) >= 2 && $length == 20) {
// We have a smaller range (xx:xx:xx:X or xx:xx:xx:xx:X)
if (substr($prefix, -2) == '28') {
$oui = substr($prefix, 0, 7);
} elseif (substr($prefix, -2) == '36') {
$oui = substr($prefix, 0, 9);
}
}
if (isset($oui)) {
// Store the OUI for the vendor in the database
DB::table('vendor_ouis')->insert([
'vendor' => $vendor,
'oui' => $oui,
]);
echo "Adding $oui for $vendor" . PHP_EOL;
}
}
} catch (Exception $e) {
echo 'Error processing Mac OUI:' . PHP_EOL;
echo 'Exception: ' . get_class($e) . PHP_EOL;
echo $e->getMessage() . PHP_EOL;
$lock->release(); // We did not succeed, so we'll try again next time
return 1;
}
} else {
echo 'Not able to acquire lock, skipping mac database update' . PHP_EOL;
return 1;
}
return 0;
}
/**
* Function to generate PeeringDB Cache
*/

View File

@ -37,11 +37,7 @@ if ($device['os'] === 'vrp') {
<tr>
<th data-column-id="port_id" data-width="100px">Port</th>
<th data-column-id="mac_address" data-width="150px" data-formatter="tooltip">MAC Address</th>
<?php
if (\LibreNMS\Config::get('mac_oui.enabled') === true) {
echo ' <th data-column-id="mac_oui" data-sortable="false" data-width="130px" data-visible="false" data-formatter="tooltip">Vendor</th>';
}
?>
<th data-column-id="mac_oui" data-sortable="false" data-width="130px" data-visible="<?php echo \LibreNMS\Config::get('mac_oui.enabled') ? 'true' : 'false' ?>" data-formatter="tooltip">Vendor</th>
<th data-column-id="ip_address" data-width="140px" data-formatter="tooltip">IP Address</th>
<th data-column-id="vlan" data-width="60px" data-formatter="tooltip"<?php echo $vlan_visibility ?>>Vlan</th>
<th data-column-id="domain" data-formatter="nac_domain" data-formatter="tooltip">Domain</th>
@ -89,7 +85,7 @@ if (\LibreNMS\Config::get('mac_oui.enabled') === true) {
"nac_authz": function (column, row) {
var value = row[column.id];
if (value === 'authorizationSuccess' || value === 'sussess') {
if (value === 'authorizationSuccess' || value === 'sussess') {
//typo in huawei MIB so we must keep sussess
return "<i class=\"fa fa-check-circle fa-lg icon-theme\" aria-hidden=\"true\" style=\"color:green;\"></i>";
} else if (value === 'authorizationFailed') {

View File

@ -5,11 +5,7 @@ $no_refresh = true;
<thead>
<tr>
<th data-column-id="mac_address" data-formatter="tooltip">MAC address</th>
<?php
if (\LibreNMS\Config::get('mac_oui.enabled') === true) {
echo ' <th data-column-id="mac_oui" data-sortable="false" data-width="" data-visible="true" data-formatter="tooltip">Vendor</th>';
}
?>
<th data-column-id="mac_oui" data-sortable="false" data-width="" data-visible="<?php echo \LibreNMS\Config::get('mac_oui.enabled') ? 'true' : 'false' ?>" data-formatter="tooltip">Vendor</th>
<th data-column-id="ipv4_address" data-formatter="tooltip">IPv4 address</th>
<th data-column-id="remote_device" data-sortable="false">Remote device</th>
<th data-column-id="remote_interface" data-sortable="false">Remote interface</th>

View File

@ -6,11 +6,7 @@ $no_refresh = true;
<thead>
<tr>
<th data-column-id="mac_address" data-width="150px" data-formatter="tooltip">MAC Address</th>
<?php
if (\LibreNMS\Config::get('mac_oui.enabled') === true) {
echo ' <th data-column-id="mac_oui" data-sortable="false" data-width="" data-visible="true" data-formatter="tooltip">Vendor</th>';
}
?>
<th data-column-id="mac_oui" data-sortable="false" data-width="" data-visible="<?php echo \LibreNMS\Config::get('mac_oui.enabled') ? 'true' : 'false' ?>" data-formatter="tooltip">Vendor</th>
<th data-column-id="ipv4_address" data-sortable="false" data-formatter="tooltip">IPv4 Address</th>
<th data-column-id="interface">Port</th>
<th data-column-id="vlan" data-width="60px">Vlan</th>

View File

@ -5,11 +5,7 @@ $no_refresh = true;
<thead>
<tr>
<th data-column-id="mac_address" data-width="150px" data-formatter="tooltip">MAC Address</th>
<?php
if (\LibreNMS\Config::get('mac_oui.enabled') === true) {
echo ' <th data-column-id="mac_oui" data-sortable="false" data-width="150px" data-visible="false" data-formatter="tooltip">Vendor</th>';
}
?>
<th data-column-id="mac_oui" data-sortable="false" data-width="150px" data-visible="<?php echo \LibreNMS\Config::get('mac_oui.enabled') ? 'true' : 'false' ?>" data-formatter="tooltip">Vendor</th>
<th data-column-id="ipv4_address" data-sortable="false" data-formatter="tooltip">IPv4 Address</th>
<th data-column-id="interface">Port</th>
<th data-column-id="description" data-formatter="tooltip">Description</th>

View File

@ -6,7 +6,7 @@
<thead>
<tr>
<th data-column-id="mac_address" data-formatter="tooltip">MAC Address</th>
<th data-column-id="mac_oui" data-sortable="false" data-visible="false" data-formatter="tooltip">Vendor</th>
<th data-column-id="mac_oui" data-sortable="false" data-visible="<?php echo \LibreNMS\Config::get('mac_oui.enabled') ? 'true' : 'false' ?>" data-formatter="tooltip">Vendor</th>
<th data-column-id="ipv4_address" data-formatter="tooltip">IP Address</th>
<th data-column-id="hostname" data-order="asc">Device</th>
<th data-column-id="interface">Interface</th>

View File

@ -7,11 +7,7 @@
<tr>
<th data-column-id="device">Device</th>
<th data-column-id="mac_address" data-width="150px" data-formatter="tooltip">MAC Address</th>
<?php
if (\LibreNMS\Config::get('mac_oui.enabled') === true) {
echo ' <th data-column-id="mac_oui" data-sortable="false" data-width="150px" data-visible="false" data-formatter="tooltip">Vendor</th>';
}
?>
<th data-column-id="mac_oui" data-sortable="false" data-width="150px" data-visible="<?php echo \LibreNMS\Config::get('mac_oui.enabled') ? 'true' : 'false' ?>" data-formatter="tooltip">Vendor</th>
<th data-column-id="ipv4_address" data-sortable="false" data-formatter="tooltip">IPv4 Address</th>
<th data-column-id="interface">Port</th>
<th data-column-id="vlan" data-width="60px">Vlan</th>

View File

@ -8,11 +8,8 @@
<th data-column-id="hostname" data-order="asc">Device</th>
<th data-column-id="interface">Interface</th>
<th data-column-id="address" data-sortable="false" data-formatter="tooltip">MAC Address</th>
<?php
if (\LibreNMS\Config::get('mac_oui.enabled') === true) {
echo ' <th data-column-id="mac_oui" data-sortable="false" data-width="150px" data-visible="false" data-formatter="tooltip">Vendor</th>';
}
?> <th data-column-id="description" data-sortable="false" data-formatter="tooltip">Description</th></tr>
<th data-column-id="mac_oui" data-sortable="false" data-width="150px" data-visible="<?php echo \LibreNMS\Config::get('mac_oui.enabled') ? 'true' : 'false' ?>" data-formatter="tooltip">Vendor</th>
<th data-column-id="description" data-sortable="false" data-formatter="tooltip">Description</th></tr>
</tr>
</thead>
</table>

View File

@ -157,6 +157,24 @@ return [
'optionValue' => 'Selected :option is invalid. Should be one of: :values',
],
],
'maintenance:fetch-ouis' => [
'description' => 'Fetch MAC OUIs and cache them to display vendor names for MAC addresses',
'options' => [
'force' => 'Ignore any settings or locks that prevent the command from being run',
'wait' => 'Wait a random amount of time, used by the scedueler to prevent server strain',
],
'disabled' => 'Mac OUI integration disabled (:setting)',
'enable_question' => 'Enable Mac OUI integration and scheduled fetching?',
'recently_fetched' => 'MAC OUI Database fetched recently, skipping update.',
'waiting' => 'Waiting :minutes minute before attempting MAC OUI update|Waiting :minutes minutes before attempting MAC OUI update',
'starting' => 'Storing Mac OUI in the database',
'downloading' => 'Downloading',
'processing' => 'Processing CSV',
'saving' => 'Saving results',
'success' => 'Successfully updated OUI/Vendor mappings. :count modified OUI|Successfully updated. :count modified OUIs',
'error' => 'Error processing Mac OUI:',
'vendor_update' => 'Adding OUI :oui for :vendor',
],
'plugin:disable' => [
'description' => 'Disable all plugins with the given name',
'arguments' => [

View File

@ -2088,9 +2088,10 @@ vendor_ouis:
Columns:
- { Field: id, Type: 'bigint unsigned', 'Null': false, Extra: auto_increment }
- { Field: vendor, Type: varchar(255), 'Null': false, Extra: '' }
- { Field: oui, Type: varchar(255), 'Null': false, Extra: '' }
- { Field: oui, Type: varchar(12), 'Null': false, Extra: '' }
Indexes:
PRIMARY: { Name: PRIMARY, Columns: [id], Unique: true, Type: BTREE }
vendor_ouis_oui_unique: { Name: vendor_ouis_oui_unique, Columns: [oui], Unique: true, Type: BTREE }
vlans:
Columns:
- { Field: vlan_id, Type: 'int unsigned', 'Null': false, Extra: auto_increment }