More more docs (#4197)

* Updated docs + some small code changes

* Added validating code doc for developers

* Doc updates

* Updates and improvements

.gitignore

@@ -22,3 +22,4 @@ nbproject
 patches
 rrd
 /vendor
+composer.phar

.travis.yml

@@ -46,5 +46,5 @@ after_success:
 script:
   - if [[ $PHP_L == 1 ]]; then find . -path './vendor' -prune -o -name "*.php" -print0 | xargs -0 -n1 -P8 php -l | grep -v '^No syntax errors detected' ; test $? -eq 1; fi
   - if [[ $PHP_L_OLD == 1 ]]; then find . -regextype posix-extended -regex "\./(lib/influxdb-php|vendor)" -prune -o -name "*.php" -print0 | xargs -0 -n1 -P8 php -l | grep -v '^No syntax errors detected' ; test $? -eq 1; fi
-  - if [[ $PHP_CS == 1 ]]; then vendor/bin/phpcs -n -p --colors --extensions=php --standard=PSR2 --ignore=html/lib/* html; fi
+  - if [[ $PHP_CS == 1 ]]; then vendor/bin/phpcs -n -p --colors --extensions=php --standard=PSR2 --ignore=html/lib/* --ignore=html/plugins/* html; fi
   - phpunit

composer.json

@@ -1,5 +1,10 @@
 {
     "require": {
         "squizlabs/php_codesniffer": "*"
+    },
+    "require-dev": {
+        "squizlabs/php_codesniffer": "*",
+        "phpunit/phpunit": "4.*"
     }
 }
+

config.php.default

@@ -9,21 +9,13 @@ $config['db_pass'] = 'PASSWORD';
 $config['db_name'] = 'librenms';
 $config['db']['extension'] = 'mysqli';// mysql or mysqli
 
-### Memcached config - We use this to store realtime usage
-$config['memcached']['enable']  = FALSE;
-$config['memcached']['host']    = 'localhost';
-$config['memcached']['port']    = 11211;
-
 // This is the user LibreNMS will run as
 //Please ensure this user is created and has the correct permissions to your install
 $config['user'] = 'librenms';
 
-### Locations - it is recommended to keep the default
-#$config['install_dir']  = "/opt/librenms";
-
 ### This should *only* be set if you want to *force* a particular hostname/port
 ### It will prevent the web interface being usable form any other hostname
-#$config['base_url']        = "http://librenms.company.com";
+$config['base_url']        = "/";
 
 ### Enable this to use rrdcached. Be sure rrd_dir is within the rrdcached dir
 ### and that your web server has permission to talk to rrdcached.
@@ -41,9 +33,6 @@ $config['auth_mechanism'] = "mysql"; # default, other options: ldap, http-auth
 #$config['nets'][] = "172.16.0.0/12";
 #$config['nets'][] = "192.168.0.0/16";
 
-# following is necessary for poller-wrapper
-# poller-wrapper is released public domain
-$config['poller-wrapper']['alerter'] = FALSE;
 # Uncomment the next line to disable daily updates
 #$config['update'] = 0;
 
@@ -55,3 +44,9 @@ $config['rrd_purge'] = 0;
 
 # Set default port association mode for new devices (default: ifIndex)
 #$config['default_port_association_mode'] = 'ifIndex';
+
+# Enable the in-built billing extension
+$config['enable_billing'] = 1;
+
+# Enable the in-built services support (Nagios plugins)
+$config['show_services'] = 1;

doc/Developing/Using-Git.md

@@ -80,11 +80,13 @@ pull request then commit away.
 ```bash
 git add path/to/new/files/or/folders
 git commit -a -m 'Added feature to do X, Y and Z'
-git checkout master
+git push origin issue-123
+```
+
+If you need to rebase against master then you can do this with:
+
+```bash
 git pull upstream master
-git push origin master
-git checkout issue-123
-git pull origin master
 git push origin issue-123
 ```
 
@@ -97,13 +99,7 @@ Now you will be ready to submit a pull request from within GitHub. To do this, g
 repo. Now select the branch you have just been working on (issue-123) from the drop down to the left and then click
 'Pull Request'. Fill in the details to describe the work you have done and click 'Create pull request'.
 
-Thanks for your first pull request :) Now, that might have been a simple update, if things get a bit more complicated
-then you will need to break down your pull request into separate commits (still a single pull request). This is usually
-done when:
-
-- You want to add / update MIBS. Do this in a separate commit including the link to where you got them from.
-- You are adding say 3 related features in one go, try and break them down into 3 separate commits.
-- Icons for new OS support need to be added as a separate commit including a link to where you got the logo from.
+Thanks for your first pull request :)
 
 Ok, that should get you started on the contributing path. If you have any other questions then stop by our IRC Channel
 on Freenode ##librenms.

doc/Developing/Validating-Code.md

@@ -0,0 +1,28 @@
+source: Developing/Validating-Code.md
+
+As part of the pull request process with GitHub we run some automated build tests to ensure that 
+the code is error free, standards [compliant](http://docs.librenms.org/Developing/Code-Guidelines/)
+and our test suite builds successfully.
+
+Rather than submit a pull request and wait for the results, you can run these checks yourself to ensure 
+a more seamless merge.
+
+> All of these commands should be run from within the librenms directory and can be run as the librenms user 
+unless otherwise noted.
+
+Install composer (you can skip this if composer is already installed).
+
+`curl -sS https://getcomposer.org/installer | php`
+
+Composer will now be installed into /opt/librenms/composer.phar.
+
+Now install the dependencies we require:
+
+`composer install`
+
+Once composer is installed you can now run the code validation script:
+
+`./scripts/pre-commit.php`
+
+If you see `Tests ok, submit away :)` then all is well. If you see other output then it should contain 
+what you need to resolve the issues and re-test.

doc/Extensions/Device-Groups.md

@@ -9,12 +9,12 @@ Patterns work in the same was as Entities within the alerting system, the format
 as __tablename.columnname__. If you are ensure of what the entity is you want then have a browse around inside MySQL using `show tables` and `desc <tablename>`.
 
 As a working example and a common question, let's assume you want to group devices by hostname. If you hostname format is dcX.[devicetype].example.com. You would use the pattern
-devices.hostname. Select the condition which in this case would Like and then enter `dc1\..*\.example.com`. This would then match dc1.sw01.example.com, dc1.rtr01.example.com but not
+`devices.hostname`. Select the condition which in this case would be `Like` and then enter `dc1\..*\.example.com`. This would then match dc1.sw01.example.com, dc1.rtr01.example.com but not
  dc2.sw01.example.com.
 
 #### Wildcards
 
-As with alerts, the `Like` operation allows RegExp.
+As with alerts, the `Like` operation allows MySQL Regular expressions.
 
 A list of common entities is maintained in our [Alerting docs](http://docs.librenms.org/Extensions/Alerting/#entities).
 
@@ -25,7 +25,7 @@ Please see our [Alerting docs](http://docs.librenms.org/Extensions/Alerting/#syn
 ### Connection
 
 If you only want to group based on one pattern then select And. If however you want to build a group based on multiple patterns then you can build a SQL like
-query using And / Or. As an example, we want to base our group on the devices hostname AND it's type. Use the pattern as before, devices.hostname, select the condition which in this case would Like and then enter dc1.@.example.com then click And. Now enter devices.type in the pattern, select Equals and enter firewall. This would then match dc1.fw01.example.com but not dc1.sw01.example.com as that is a network type.
+query using And / Or. As an example, we want to base our group on the devices hostname AND it's type. Use the pattern as before, `devices.hostname`, select the condition which in this case would be `Like` and then enter `dc1.@.example.com` then click And. Now enter `devices.type` in the pattern, select `Equals` and enter `firewall`. This would then match dc1.fw01.example.com but not dc1.sw01.example.com as that is a network type.
 
 You can now select this group from the Devices -> All Devices link in the navigation at the top. You can also use the group to map alert rules to by creating an alert mapping
-Overview -> Alerts -> Rule Mapping.
+`Overview -> Alerts -> Rule Mapping`.

doc/Extensions/Globe-Frontpage.md

@@ -1,23 +1,21 @@
 source: Extensions/Globe-Frontpage.md
 # Globe Frontpage Configuration
 
-LibreNMS comes with a configurable geochart based frontpage to visualize where your gear is located geographically.
+LibreNMS comes with a configurable geochart based widget to visualize where your equipment is located geographically.
 
-### Experimental map
+### World Map
 
-An new experimental map is available, this requires you to have properly formatted addresses in sysLocation or sysLocation override. As part of the standard poller these addresses will be Geocoded by Google and stored in the database. To enable this please set the following config:
+An new map is available, this requires you to have properly formatted addresses in sysLocation or sysLocation override. As part of the standard poller these addresses will be Geocoded by Google and stored in the database. To enable this please set the following config:
 
 ```php
-$config['front_page']       = "pages/front/map.php";
 $config['geoloc']['latlng'] = true;
 $config['geoloc']['engine'] = "google";//Only one available at present
 ```
 
 Location resolution happens as follows (when `$config['geoloc']['latlng'] == true;`):
- 1. if `device['location']` contains `[lat, lng]` (note the square brackets), that is used
- 1. if there is a location overide in the `locations` table where `locations.location == device['location']`, that is used
-  * currently, no web UI
- 1. attempt to resolve lat, lng using `$config['geoloc']['engine']`
+ 1. If `device['location']` contains `[lat, lng]` (note the square brackets), that is used
+ 1. If there is a location overide for the device in the WebUI and it contains `[lat, lng]` (note the square brackets), that is used.
+ 1. Attempt to resolve lat, lng using `$config['geoloc']['engine']`
 
 We have two current mapping engines available:
 
@@ -56,13 +54,3 @@ $config['mapael']['default_zoom'] = 20;
 ```
 
 A list of maps can be found in html/js/maps/ or html/js/mapael-maps/.
-
-### Standard Globe map
-
-To enable it, set `$config['front_page'] = "pages/front/globe.php";` in your `config.php`.
-
-You can use any of these config-parameters to adjust some aspects of it:
-
-- `$config['frontpage_globe']['markers']`    is used to change what is being shown on the Markers of the map. It can be either `devices` or `ports`
-- `$config['frontpage_globe']['region']`     defines the Region to chart. Any region supported by Google's GeoChart API is allowed (https://developers.google.com/chart/interactive/docs/gallery/geochart#continent-hierarchy-and-codes)
-- `$config['frontpage_globe']['resolution']` can be 'countries', 'provinces' or 'metros' (latter two are mostly US only due to google-limits).

doc/Extensions/Memcached.md

@@ -12,6 +12,9 @@ $config['memcached']['port']    = 11211;
 ```
 
 By default values are kept for 4 Minutes inside the memcached, you can adjust this retention time by modifying the `$config['memcached']['ttl']` value to any desired amount of seconds.
+
+> This means that you can see what appears to be stale data for up to 4 minutes. If you edit an alert rule for example then those changes may not show immediately.
+
 It's strongly discouraged to set this above `300` (5 Minutes) to avoid interference with the polling, discovery and alerting processes.
 
 If you use the Distributed Poller, you can point this to the same memcached instance. However a local memcached will perform better in any case.

doc/Extensions/Oxidized.md

@@ -57,13 +57,13 @@ LibreNMS is able to reload the Oxidized list of nodes, each time a device is add
 To do so, edit the option in Global Settings>External Settings>Oxidized Integration or add the following to your config.php.
 
 ```php
-$config['oxidized']['reload_nodes'] = TRUE;
+$config['oxidized']['reload_nodes'] = true;
 
 ```
 
 #### Using Groups
 
-To return a group to Oxidized you can do this by matching a regex for either hostname, os or location. The order is hostname is matched first, if nothing is found then os is tried and then location is attempted.
+To return a group to Oxidized you can do this by matching a regex for either `hostname`, `os` or `location`. The order is `hostname` is matched first, if nothing is found then `os` is tried and then `location` is attempted.
 The first match found will be used. To match on the device hostnames that contain 'lon-sw' or if the location contains 'London' then you would place the following within config.php:
 
 ```php

doc/Extensions/Plugin-System.md

@@ -1,6 +1,8 @@
 source: Extensions/Plugin-System.md
 # Developing for the Plugin System
 
+> This will most likely be deprecated in favour of adding the possible extensions to the core code base.
+
 This documentation will hopefully give you a basis for how to write a plugin for LibreNMS.
 
 A test plugin is available on GitHub: https://github.com/laf/Test

doc/General/Acknowledgement.md

@@ -35,5 +35,3 @@ LibreNMS 3rd party acknowledgements
   - html/graph-realtime.php: BSD (original?)
   - html/includes/collectd/: GPLv2 only
   - overLIB (html/js/overlib_mini.js): modified Artistic 1.0?
-  - scripts/*/mysql: GPLv2 only
-  - check_mk (scripts/check_mk*): GPLv2

doc/General/Contributing.md

@@ -34,9 +34,9 @@ To agree with these assertions, please submit a GitHub pull request against
 [AUTHORS.md][5], adding or altering a **single line** *containing your name,
 email address, and GitHub user id* in the file (so that it can be matched to
 your commits), and stating in the *commit log* (not the pull request text):
+
 ```
-	I agree to the conditions of the Contributor Agreement
-	contained in doc/General/Contributing.md.
+I agree to the conditions of the Contributor Agreement contained in doc/General/Contributing.md.
 ```
 
 Local patches
@@ -61,8 +61,6 @@ the package information to the header.
 ```
 <?php
 /*
- * Copyright (c) 2015 Internet Widgitz Pty Ltd <http://example.com/>
- *
  * This program is free software: you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
  * Free Software Foundation, either version 3 of the License, or (at your

doc/General/Credits.md

@@ -50,9 +50,3 @@ Other components (needs details filled in):
 - html/includes/collectd/: GPLv2 only
 
 - overLIB (html/js/overlib_mini.js): modified Artistic 1.0?
-
-- scripts/*/mysql: GPLv2 only
-
-- check_mk (scripts/observium_agent*): GPLv2
-
-- qTip (html/css/jquery.qtip.min.css and html/js/qtip/jquery.qtip.min.js): GPLv2

doc/General/Updating.md

@@ -4,7 +4,7 @@ source: General/Updating.md
 If you would like to perform a manual update
 then you can do this by running the following command as the **librenms** user:
 
-	./daily.sh
+`./daily.sh`
 
 This will update both the core LibreNMS files but also update the database
 structure if updates are available.
@@ -13,12 +13,12 @@ structure if updates are available.
 LibreNMS follows the master branch on github for daily updates.
 You can change to the monthly releases by setting:
 
-    $config['update_channel'] = 'release';
+`$config['update_channel'] = 'release';`
 
 ## Disabling automatic updates ##
 LibreNMS by default performs updates on a daily basis. This can be disabled
 by ensuring:
 
-	$config['update'] = 0;
+`$config['update'] = 0;`
 
 is no longer commented out. 

doc/General/Welcome-to-Observium-users.md

@@ -59,8 +59,6 @@ Reasons why you might want to use LibreNMS instead of Observium:
 "LibreNMS README"
 [3]: http://fisheye.observium.org/rdiff/Observium?csid=3251&u&N
 "Link to Observium license change"
-[4]: https://github.com/librenms/librenms/blob/master/doc/General/Roadmap.md
-"LibreNMS ROADMAP"
 [5]: https://github.com/librenms/librenms/blob/master/LICENSE.txt
 "LibreNMS copy of GPL v3"
 [6]: http://www.gnu.org/philosophy/free-sw.html

doc/Support/Discovery Support.md

@@ -6,18 +6,21 @@ This document will explain how to use discovery.php to debug issues or manually
 #### Command options
 ```bash
 -h <device id> | <device hostname wildcard>  Poll single device
-   -h odd                                       Poll odd numbered devices  (same as -i 2 -n 0)
-   -h even                                      Poll even numbered devices (same as -i 2 -n 1)
-   -h all                                       Poll all devices
-   -h new                                       Poll all devices that have not had a discovery run before
-
-   -i <instances> -n <number>                   Poll as instance <number> of <instances>
+-h odd                                       Poll odd numbered devices  (same as -i 2 -n 0)
+-h even                                      Poll even numbered devices (same as -i 2 -n 1)
+-h all                                       Poll all devices
+-h new                                       Poll all devices that have not had a discovery run before
+--os <os_name>                               Poll devices only with specified operating system
+--type <type>                                Poll devices only with specified type
+-i <instances> -n <number>                   Poll as instance <number> of <instances>
                                              Instances start at 0. 0-3 for -n 4
 
+Debugging and testing options:
+-d                                           Enable debugging output
+-v                                           Enable verbose debugging output
+-m                                           Specify single module to be run
+
 
-   Debugging and testing options:
-   -d                                           Enable debugging output
-   -m                                           Specify single module to be run
 ```
 
 `-h` Use this to specify a device via either id or hostname (including wildcard using *). You can also specify odd and
@@ -26,15 +29,16 @@ new will poll only those devices that have recently been added or have been sele
 
 `-i` This can be used to stagger the discovery process.
 
-`-d` Enables debugging output (verbose output) so that you can see what is happening during a discovery run. This includes
-things like rrd updates, SQL queries and response from snmp.
+`-d` Enables debugging output (verbose output but with most sensitive data masked) so that you can see what is happening during a discovery run. This includes things like rrd updates, SQL queries and response from snmp.
+
+`-v` Enables verbose debugging output with all data in tact.
 
 `-m` This enables you to specify the module you want to run for discovery.
 
 #### Discovery config
 
 These are the default discovery config items. You can globally disable a module by setting it to 0. If you just want to
-disable it for one device then you can do this within the WebUI -> Settings -> Modules.
+disable it for one device then you can do this within the WebUI -> Device -> Settings -> Modules.
 
 ```php
 $config['discovery_modules']['os']                        = 1;
@@ -160,15 +164,12 @@ Multiple Modules
 ./discovery.php -h localhost -m ports,entity-physical -d
 ```
 
-It is then advisable to sanitise the output before pasting it somewhere as the debug output will contain snmp details
-amongst other items including port descriptions.
+Using `-d` shouldn't output much sensitive information, `-v` will so it is then advisable to sanitise the output before pasting it somewhere as the debug output will contain snmp details amongst other items including port descriptions.
 
 The output will contain:
 
 DB Updates
 
-RRD Updates
-
 SNMP Response
 
 ### SNMP Scan

doc/Support/Install Validation.md

@@ -2,7 +2,7 @@ source: Install Validation.md
 Install validation
 ------------------
 
-With a lot of configuration possibilities and at present the only way to do this being by manually editing config.php then it's not
+With a lot of configuration possibilities, manually editing config.php means it's not
 uncommon that mistakes get made. It's also impossible to validate user input in config.php when you're just using a text editor :)
 
 So, to try and help with some of the general issues people come across we've put together a simple validation tool which at present will:
@@ -22,10 +22,12 @@ Optionally you can also pass -m and a module name for that to be tested. Current
  - dist-poller - This will test your distributed poller configuration.
  - rrdcheck - This will test your rrd files to see if they are unreadable or corrupted (source of broken graphs).
 
-Output, this is color coded to try and make things a little easier:
+You can run validate.php as `root` by executing `./validate.php` within your install directory.
 
-Green OK - This is a good thing, you can skip over these :)
+The output will provide you either a clean bill of health or a list of things you need to fix:
 
-Yellow WARN - You probably want to check this out.
+OK - This is a good thing, you can skip over these :)
 
-Red FAIL - This is going to need your attention!
+WARN - You probably want to check this out.
+
+FAIL - This is going to need your attention!

doc/Support/Poller Support.md

@@ -17,7 +17,9 @@ This document will explain how to use poller.php to debug issues or manually run
 
 Debugging and testing options:
 -r                                           Do not create or update RRDs
+-f                                           Do not insert data into InfluxDB
 -d                                           Enable debugging output
+-v                                           Enable verbose debugging output
 -m                                           Specify module(s) to be run
 ```
 
@@ -28,8 +30,9 @@ even. all will run poller against all devices.
 
 `-r` This option will suppress the creation or update of RRD files.
 
-`-d` Enables debugging output (verbose output) so that you can see what is happening during a poller run. This includes
-things like rrd updates, SQL queries and response from snmp.
+`-d` Enables debugging output (verbose output but with most sensitive data masked) so that you can see what is happening during a poller run. This includes things like rrd updates, SQL queries and response from snmp.
+
+`-v` Enables verbose debugging output with all data in tact.
 
 `-m` This enables you to specify the module you want to run for poller.
 
@@ -170,8 +173,7 @@ Multiple Modules
 ./poller.php -h localhost -m ports,entity-physical -d
 ```
 
-It is then advisable to sanitise the output before pasting it somewhere as the debug output will contain snmp details
-amongst other items including port descriptions.
+Using `-d` shouldn't output much sensitive information, `-v` will so it is then advisable to sanitise the output before pasting it somewhere as the debug output will contain snmp details amongst other items including port descriptions.
 
 The output will contain:
 

doc/Support/Support-New-OS.md

@@ -105,8 +105,6 @@ Polling
 
 At this step we should see all the values retrieved in LibreNMS.
 
-
-
 #### FULL SUPPORT FOR A NEW OS
 
 ### MIB

html/install.php

@@ -397,9 +397,6 @@ $config_file = <<<"EOD"
 #\$config\['nets'\]\[\] = "172.16.0.0/12";
 #\$config\['nets'\]\[\] = "192.168.0.0/16";
 
-# following is necessary for poller-wrapper
-# poller-wrapper is released public domain
-\$config\['poller-wrapper'\]\['alerter'\] = FALSE;
 # Uncomment the next line to disable daily updates
 #\$config\['update'\] = 0;
 EOD;

librenms.cron

@@ -1,4 +1,5 @@
 # It's recommended not to run this cron anymore - please see librenms.nonroot.cron
+# PLEASE DO NOT USE THIS FILE
 
 33  */6   * * *   root    /opt/librenms/discovery.php -h all >> /dev/null 2>&1
 */5  *    * * *   root    /opt/librenms/discovery.php -h new >> /dev/null 2>&1

librenms.nonroot.cron

@@ -5,3 +5,7 @@
 */5  *    * * *   librenms    /opt/librenms/cronic /opt/librenms/poller-wrapper.py 16
 15   0    * * *   librenms    /opt/librenms/daily.sh >> /dev/null 2>&1
 *    *    * * *   librenms    /opt/librenms/alerts.php >> /dev/null 2>&1
+*/5  *    * * *   librenms    /opt/librenms/poll-billing.php >> /dev/null 2>&1
+01   *    * * *   librenms    /opt/librenms/billing-calculate.php >> /dev/null 2>&1
+*/5  *    * * *   librenms    /opt/librenms/check-services.php >> /dev/null 2>&1
+

scripts/pre-commit.php

@@ -0,0 +1,36 @@
+#!/usr/bin/env php
+<?php
+
+$failed = false;
+
+if (version_compare(PHP_VERSION, '5.6') >= 0) {
+    $lint = `find . -path './vendor' -prune -o -name "*.php" -print0 | xargs -0 -n1 -P8 php -l | grep -v '^No syntax errors detected' ; test $? -eq 1`;
+} else {
+    $lint = `find . -regextype posix-extended -regex "\./(lib/influxdb-php|vendor)" -prune -o -name "*.php" -print0 | xargs -0 -n1 -P8 php -l | grep -v '^No syntax errors detected' ; test $? -eq 1`;
+}
+
+if (!empty($lint)) {
+    echo "lint check has failed\n";
+    print_r($lint);
+    $failed = true;
+}
+
+$phpcs = `./vendor/bin/phpcs -n -p --colors --extensions=php --standard=PSR2 --ignore=html/lib/* --ignore=html/plugins/ html`;
+
+if (!empty($phpcs)) {
+    echo "PSR2 check has failed\n";
+    print_r($phpcs);
+    $failed = true;
+}
+
+$phpunit = `./vendor/bin/phpunit`;
+
+if(!strstr($phpunit, "OK")) {
+    echo "phpunit tests have failed\n";
+    print_r($phpunit);
+    $failed = true;
+}
+
+if ($failed === false) {
+    echo "Tests ok, submit away :)\n";
+}