FDB table with history capabilities (#9804)

* Migration script and data feeding

* Adding the columns in the GUI

* build schema

* update FDB test data

* Use of timestamps()

* ignore created_at and updated_at in tests, and regenerate test impacted

* daily.sh does the cleaning

* space cleaning codeclimate

* Use carbon instead of str-val

* handle when $fdb_entry->updated_at and created at are null

* handle when $fdb_entry->updated_at and created at are null (force travis rerun)

* Doc update

* Doc update

app/Http/Controllers/Table/FdbTablesController.php

@@ -45,7 +45,7 @@ class FdbTablesController extends TableController
         return [
             'port_id' => 'nullable|integer',
             'device_id' => 'nullable|integer',
-            'serachby' => 'in:mac,vlan,dnsname,ip,description',
+            'serachby' => 'in:mac,vlan,dnsname,ip,description,first_seen,last_seen',
             'dns' => 'nullable|in:true,false',
         ];
     }
@@ -137,6 +137,14 @@ class FdbTablesController extends TableController
                 ->orderBy('ports.ifDescr', $sort['description']);
         }
 
+        if (isset($sort['last_seen'])) {
+            $query->orderBy('updated_at', $sort['last_seen']);
+        }
+
+        if (isset($sort['first_seen'])) {
+            $query->orderBy('created_at', $sort['first_seen']);
+        }
+
         return $query;
     }
 
@@ -152,7 +160,17 @@ class FdbTablesController extends TableController
             'vlan' => $fdb_entry->vlan ? $fdb_entry->vlan->vlan_vlan : '',
             'description' => '',
             'dnsname' => $ip_info['dns'],
+            'first_seen' => 'unknown',
+            'last_seen' => 'unknown'
         ];
+        
+        // diffForHumans and doDateTimeString are not safe
+        if ($fdb_entry->updated_at) {
+            $item['last_seen'] = $fdb_entry->updated_at->diffForHumans();
+        }
+        if ($fdb_entry->created_at) {
+            $item['first_seen'] = $fdb_entry->created_at->toDateTimeString();
+        }
 
         if ($fdb_entry->port) {
             $item['interface'] = Url::portLink($fdb_entry->port, $fdb_entry->port->getShortLabel());

app/Models/PortsFdb.php

@@ -6,7 +6,7 @@ class PortsFdb extends PortRelatedModel
 {
     protected $table = 'ports_fdb';
     protected $primaryKey = 'ports_fdb_id';
-    public $timestamps = false;
+    public $timestamps = true;
 
     // ---- Define Relationships ----
 

daily.php

@@ -90,6 +90,11 @@ if ($options['f'] === 'syslog') {
     }
 }
 
+if ($options['f'] === 'ports_fdb') {
+    $ret = lock_and_purge('ports_fdb', 'updated_at < DATE_SUB(NOW(), INTERVAL ? DAY)');
+    exit($ret);
+}
+
 if ($options['f'] === 'eventlog') {
     $ret = lock_and_purge('eventlog', 'datetime < DATE_SUB(NOW(), INTERVAL ? DAY)');
     exit($ret);

daily.sh

@@ -273,6 +273,7 @@ main () {
                                "bill_data"
                                "alert_log"
                                "rrd_purge"
+                               "ports_fdb"
                                "ports_purge");
                 call_daily_php "${options[@]}";
             ;;

database/migrations/2019_02_10_220000_add_dates_to_fdb.php

@@ -0,0 +1,39 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use App\Models\PortsFdb;
+use Carbon\Carbon;
+
+class AddDatesToFdb extends Migration
+{
+
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('ports_fdb', function (Blueprint $table) {
+              $table->timestamps();
+        });
+        
+        // Let's get a value for existing PortsFdb data :
+        DB::table('ports_fdb')->update(array('created_at' => \Carbon\Carbon::now()));
+        DB::table('ports_fdb')->update(array('updated_at' => \Carbon\Carbon::now()));
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('ports_fdb', function (Blueprint $table) {
+            $table->dropColumn('created_at');
+            $table->dropColumn('updated_at');
+        });
+    }
+}

doc/Support/Configuration.md

@@ -629,9 +629,11 @@ $config['eventlog_purge']                                 = 30;
 $config['authlog_purge']                                  = 30;
 $config['perf_times_purge']                               = 30;
 $config['device_perf_purge']                              = 7;
+$config['alert_log_purge']                                = 365;
+$config['port_fdb_purge']                                 = 10;
 $config['rrd_purge']                                      = 90;// Not set by default
 ```
-These options will ensure data within LibreNMS over X days old is automatically purged. You can alter these individually,
+These options will ensure data within LibreNMS over X days old is automatically purged. You can alter these individually.
 values are in days.
 
 > NOTE: Please be aware that `$config['rrd_purge']` is _NOT_ set by default. This option will remove any old data within 

doc/Support/Discovery Support.md

@@ -143,6 +143,8 @@ $config['os']['linux']['discovery_modules']['discovery-arp'] = true;
 
 `arp-table`: Detection of the ARP table for the device.
 
+`fdb-table`: Detection of the Forwarding DataBase table for the device, with history data.
+
 `discovery-arp`: Auto discovery via ARP.
 
 `junose-atm-vp`: Juniper ATM support.

doc/Support/Features.md

@@ -5,11 +5,13 @@ path: blob/master/doc/
 Here's a brief list of supported features, some might be missing.
 If you think something is missing, feel free to ask us.
 
+* Auto discovery
 * Alerting
-* API
-* Auto Updating
+* Multiple environnement sensors support
+* Multiple protocols data collection (STP, OSPF, BGP etc)
+* VLAN, ARP and FDB table collection
 * Customizable Dashboards
-* Device Backup (Oxidized, RANCID)
+* Device Backup integration (Oxidized, RANCID)
 * Distributed Polling
 * Multiple Authentication Methods (MySQL, LDAP, Active Directory, HTTP)
 * NetFlow, sFlow, IPFIX (NfSen)
@@ -17,6 +19,8 @@ If you think something is missing, feel free to ask us.
 * Syslog (Integrated, Graylog)
 * Traffic Billing (Quota, 95th Percentile)
 * Two Factor Authentication
+* API
+* Auto Updating
 
 ### Vendors
 Here's a brief list of supported vendors, some might be missing.
@@ -34,6 +38,7 @@ If you are unsure of whether your device is supported or not, feel free to ask u
 * Areca
 * Arista
 * Aruba
+* Audiocodes
 * Avaya
 * Avocent
 * AVTECH
@@ -42,6 +47,7 @@ If you are unsure of whether your device is supported or not, feel free to ask u
 * BCM963
 * BDCom
 * Bintec
+* Bluecat
 * BNT
 * Brocade
 * Brother
@@ -62,7 +68,9 @@ If you are unsure of whether your device is supported or not, feel free to ask u
 * Dell
 * Delta Power Solutions
 * DEVELOP
+* DKT Comega
 * Eaton
+* EDS
 * EMC
 * Engenius
 * Enterasys
@@ -70,6 +78,7 @@ If you are unsure of whether your device is supported or not, feel free to ask u
 * Extreme Networks
 * F5
 * Fastback Networks
+* Fiberstore
 * FiberHome
 * Force10
 * Fortinet
@@ -108,6 +117,7 @@ If you are unsure of whether your device is supported or not, feel free to ask u
 * Netopia
 * NetVision
 * NetWare
+* Nokia
 * NRG
 * OKI
 * OpenBSD

html/pages/device/port/fdb.inc.php

@@ -9,6 +9,8 @@ $no_refresh = true;
             <th data-column-id="interface">Port</th>
             <th data-column-id="vlan" data-width="60px">Vlan</th>
             <th data-column-id="dnsname" data-sortable="false" data-visible="false">DNS Name</th>
+            <th data-column-id="first_seen" data-width="165px">First seen</th>
+            <th data-column-id="last_seen" data-width="165px">Last seen</th>
         </tr>
     </thead>
 </table>

html/pages/device/ports/fdb.inc.php

@@ -10,6 +10,8 @@ $no_refresh = true;
             <th data-column-id="description">Description</th>
             <th data-column-id="vlan" data-width="60px">Vlan</th>
             <th data-column-id="dnsname" data-sortable="false" data-visible="false">DNS Name</th>
+            <th data-column-id="first_seen" data-width="165px">First seen</th>
+            <th data-column-id="last_seen" data-width="165px">Last seen</th>
         </tr>
     </thead>
 </table>

html/pages/search/fdb.inc.php

@@ -12,6 +12,8 @@
                 <th data-column-id="vlan" data-width="60px">Vlan</th>
                 <th data-column-id="description">Description</th>
                 <th data-column-id="dnsname" data-sortable="false" data-visible="false">DNS Name</th>
+                <th data-column-id="first_seen" data-width="165px">First seen</th>
+                <th data-column-id="last_seen" data-width="165px">Last seen</th>
             </tr>
         </thead>
     </table>

includes/defaults.inc.php

@@ -852,6 +852,8 @@ $config['device_perf_purge'] = 7;
 // Number in days of how long to keep device performance data for.
 $config['alert_log_purge'] = 365;
 // Number in days of how long to keep alert log data for.
+$config['ports_fdb_purge'] = 10;
+// Number in days of how long to keep fdb table data for.
 
 // Date format for PHP date()s
 $config['dateformat']['long'] = 'r';

includes/discovery/fdb-table.inc.php

@@ -39,15 +39,20 @@ if (!empty($insert)) {
 
                 if ($existing_fdbs[$vlan_id][$mac_address_entry]['port_id'] != $new_port) {
                     $port_fdb_id = $existing_fdbs[$vlan_id][$mac_address_entry]['ports_fdb_id'];
-
                     dbUpdate(
-                        array('port_id' => $new_port),
+                        array('port_id' => $new_port, 'updated_at' => array('NOW()'),),
                         'ports_fdb',
                         '`device_id` = ? AND `vlan_id` = ? AND `mac_address` = ?',
                         array($device['device_id'], $vlan_id, $mac_address_entry)
                     );
                     echo 'U';
                 } else {
+                    dbUpdate(
+                        array('updated_at' => array('NOW()'),), //we need to do this unless we use Eloquent "update" method
+                        'ports_fdb',
+                        '`device_id` = ? AND `vlan_id` = ? AND `mac_address` = ?',
+                        array($device['device_id'], $vlan_id, $mac_address_entry)
+                    );
                     echo '.';
                 }
                 unset($existing_fdbs[$vlan_id][$mac_address_entry]);
@@ -57,6 +62,8 @@ if (!empty($insert)) {
                     'mac_address' => $mac_address_entry,
                     'vlan_id' => $vlan_id,
                     'device_id' => $device['device_id'],
+                    'created_at' => array('NOW()'), //we need to do this unless we use Eloquent "create" method
+                    'updated_at' => array('NOW()'), //we need to do this unless we use Eloquent "update" method
                 );
 
                 dbInsert($new_entry, 'ports_fdb');
@@ -67,17 +74,9 @@ if (!empty($insert)) {
         echo PHP_EOL;
     }
 
+    //We do not delete anything here, as daily.sh will take care of the cleaning.
+
     // Delete old entries from the database
-    foreach ($existing_fdbs as $vlan_id => $entries) {
-        foreach ($entries as $entry) {
-            dbDelete(
-                'ports_fdb',
-                '`port_id` = ? AND `mac_address` = ? AND `vlan_id` = ? and `device_id` = ?',
-                array($entry['port_id'], $entry['mac_address'], $entry['vlan_id'], $entry['device_id'])
-            );
-            d_echo("Deleting: {$entry['mac_address']}\n", '-');
-        }
-    }
 }
 
 unset(

misc/db_schema.yaml

@@ -1241,6 +1241,8 @@ ports_fdb:
     - { Field: mac_address, Type: varchar(32), 'Null': false, Extra: '' }
     - { Field: vlan_id, Type: 'int(10) unsigned', 'Null': false, Extra: '' }
     - { Field: device_id, Type: 'int(10) unsigned', 'Null': false, Extra: '' }
+    - { Field: created_at, Type: timestamp, 'Null': true, Extra: '' }
+    - { Field: updated_at, Type: timestamp, 'Null': true, Extra: '' }
   Indexes:
     PRIMARY: { Name: PRIMARY, Columns: [ports_fdb_id], Unique: true, Type: BTREE }
     mac_address: { Name: mac_address, Columns: [mac_address], Unique: false, Type: BTREE }

tests/data/edgecos_es3510ma.json

@@ -10,11 +10,11 @@
                     "version": null,
                     "hardware": null,
                     "features": null,
-                    "location": null,
                     "os": "edgecos",
                     "type": "network",
                     "serial": null,
-                    "icon": "edge-core.png"
+                    "icon": "edge-core.png",
+                    "location": null
                 }
             ]
         },
@@ -28,11 +28,11 @@
                     "version": " ",
                     "hardware": null,
                     "features": null,
-                    "location": "<private>",
                     "os": "edgecos",
                     "type": "network",
                     "serial": null,
-                    "icon": "edge-core.png"
+                    "icon": "edge-core.png",
+                    "location": "<private>"
                 }
             ]
         }

tests/module_tables.yaml

@@ -21,7 +21,7 @@ bgp-peers:
         excluded_fields: [device_id]
 fdb-table:
     ports_fdb:
-        excluded_fields: [device_id, ports_fdb_id, port_id, vlan_id]
+        excluded_fields: [device_id, ports_fdb_id, port_id, vlan_id, created_at, updated_at]
         joins:
             - { left: ports_fdb.port_id, right: ports.port_id, select: [ifIndex] }
             - { left: ports_fdb.vlan_id, right: vlans.vlan_id, select: [vlan_vlan] }