service watchdog - add systemd watchdog for resiliency (#12188)
* Add systemd watchdog service * Add systemd watchdog service * Add systemd watchdog service - add try * Add systemd watchdog service - add try * Add systemd watchdog service - add try * Add systemd watchdog service - add try * Add systemd watchdog service - add try * Add systemd watchdog service - update docs for python3-systemd * systemd-watchdog - move to 10 second alert frequency * systemd-watchdog - move to 10 second alert frequency * systemd-watchdog - move to 30 second restart, 10 second delay between restarts * systemd-watchdog - safely integrate changes * systemd-watchdog - safely integrate changes * systemd-watchdog - revert old doc changes * systemd-watchdog - doc typo fix
This commit is contained in:
Anthony F. McInerney
GitHub
parent
54f1ac2f67
commit
a625faaa1b
|
|
@ -23,6 +23,12 @@ from socket import gethostname
|
|||
from signal import signal, SIGTERM, SIGQUIT, SIGINT, SIGHUP, SIGCHLD, SIG_DFL
|
||||
from uuid import uuid1
|
||||
|
||||
try:
|
||||
from systemd.daemon import notify
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
class ServiceConfig:
|
||||
def __init__(self):
|
||||
|
|
@ -290,6 +296,7 @@ class Service:
|
|||
self.watchdog_timer = LibreNMS.RecurringTimer(self.config.poller.frequency, self.logfile_watchdog, 'watchdog')
|
||||
else:
|
||||
info("Watchdog is disabled.")
|
||||
self.systemd_watchdog_timer = LibreNMS.RecurringTimer(10, self.systemd_watchdog, 'systemd-watchdog')
|
||||
self.is_master = False
|
||||
|
||||
def service_age(self):
|
||||
|
|
@ -353,6 +360,7 @@ class Service:
|
|||
if self.config.update_enabled:
|
||||
self.daily_timer.start()
|
||||
self.stats_timer.start()
|
||||
self.systemd_watchdog_timer.start()
|
||||
if self.config.watchdog_enabled:
|
||||
self.watchdog_timer.start()
|
||||
|
||||
|
|
@ -577,6 +585,7 @@ class Service:
|
|||
|
||||
self.daily_timer.stop()
|
||||
self.stats_timer.stop()
|
||||
self.systemd_watchdog_timer.stop()
|
||||
if self.config.watchdog_enabled:
|
||||
self.watchdog_timer.stop()
|
||||
|
||||
|
|
@ -668,6 +677,10 @@ class Service:
|
|||
except pymysql.err.Error:
|
||||
exception("Unable to log performance statistics - is the database still online?")
|
||||
|
||||
def systemd_watchdog(self):
|
||||
if 'systemd.daemon' in sys.modules:
|
||||
notify("WATCHDOG=1")
|
||||
|
||||
def logfile_watchdog(self):
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -177,6 +177,16 @@ install run `cp /opt/librenms/misc/librenms.service
|
|||
/etc/systemd/system/librenms.service && systemctl enable --now
|
||||
librenms.service`
|
||||
|
||||
## systemd-watchdog
|
||||
|
||||
A systemd unit file can be found in `misc/librenms-watchdog.service`. To
|
||||
install run `cp /opt/librenms/misc/librenms-watchdog.service
|
||||
/etc/systemd/system/librenms.service && systemctl enable --now
|
||||
librenms.service`
|
||||
|
||||
This requires: python3-systemd (or python-systemd on older systems)
|
||||
or https://pypi.org/project/systemd-python/
|
||||
|
||||
## OS-Specific Instructions
|
||||
|
||||
### RHEL/CentOS
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ Connect to the server command line and follow the instructions below.
|
|||
apt install software-properties-common
|
||||
add-apt-repository universe
|
||||
apt update
|
||||
apt install acl curl composer fping git graphviz imagemagick mariadb-client mariadb-server mtr-tiny nginx-full nmap php7.4-cli php7.4-curl php7.4-fpm php7.4-gd php7.4-json php7.4-mbstring php7.4-mysql php7.4-snmp php7.4-xml php7.4-zip rrdtool snmp snmpd whois unzip python3-pymysql python3-dotenv python3-redis python3-setuptools
|
||||
apt install acl curl composer fping git graphviz imagemagick mariadb-client mariadb-server mtr-tiny nginx-full nmap php7.4-cli php7.4-curl php7.4-fpm php7.4-gd php7.4-json php7.4-mbstring php7.4-mysql php7.4-snmp php7.4-xml php7.4-zip rrdtool snmp snmpd whois unzip python3-pymysql python3-dotenv python3-redis python3-setuptools python3-systemd
|
||||
```
|
||||
|
||||
=== "Apache"
|
||||
|
|
@ -32,7 +32,7 @@ Connect to the server command line and follow the instructions below.
|
|||
apt install software-properties-common
|
||||
add-apt-repository universe
|
||||
apt update
|
||||
apt install acl curl apache2 composer fping git graphviz imagemagick libapache2-mod-fcgid mariadb-client mariadb-server mtr-tiny nmap php7.4-cli php7.4-curl php7.4-fpm php7.4-gd php7.4-json php7.4-mbstring php7.4-mysql php7.4-snmp php7.4-xml php7.4-zip rrdtool snmp snmpd whois python3-pymysql python3-dotenv python3-redis python3-setuptools
|
||||
apt install acl curl apache2 composer fping git graphviz imagemagick libapache2-mod-fcgid mariadb-client mariadb-server mtr-tiny nmap php7.4-cli php7.4-curl php7.4-fpm php7.4-gd php7.4-json php7.4-mbstring php7.4-mysql php7.4-snmp php7.4-xml php7.4-zip rrdtool snmp snmpd whois python3-pymysql python3-dotenv python3-redis python3-setuptools python3-systemd
|
||||
```
|
||||
|
||||
=== "CentOS 8"
|
||||
|
|
@ -41,7 +41,7 @@ Connect to the server command line and follow the instructions below.
|
|||
dnf -y install epel-release
|
||||
dnf module reset php
|
||||
dnf module enable php:7.3
|
||||
dnf install bash-completion cronie fping git ImageMagick mariadb-server mtr net-snmp net-snmp-utils nginx nmap php-fpm php-cli php-common php-curl php-gd php-json php-mbstring php-process php-snmp php-xml php-zip php-mysqlnd python3 python3-PyMySQL python3-redis python3-memcached python3-pip rrdtool unzip
|
||||
dnf install bash-completion cronie fping git ImageMagick mariadb-server mtr net-snmp net-snmp-utils nginx nmap php-fpm php-cli php-common php-curl php-gd php-json php-mbstring php-process php-snmp php-xml php-zip php-mysqlnd python3 python3-PyMySQL python3-redis python3-memcached python3-pip python3-systemd rrdtool unzip
|
||||
```
|
||||
|
||||
=== "Apache"
|
||||
|
|
@ -49,13 +49,13 @@ Connect to the server command line and follow the instructions below.
|
|||
dnf -y install epel-release
|
||||
dnf module reset php
|
||||
dnf module enable php:7.3
|
||||
dnf install bash-completion cronie fping git httpd ImageMagick mariadb-server mtr net-snmp net-snmp-utils nmap php-fpm php-cli php-common php-curl php-gd php-json php-mbstring php-process php-snmp php-xml php-zip php-mysqlnd python3 python3-PyMySQL python3-redis python3-memcached python3-pip rrdtool unzip
|
||||
dnf install bash-completion cronie fping git httpd ImageMagick mariadb-server mtr net-snmp net-snmp-utils nmap php-fpm php-cli php-common php-curl php-gd php-json php-mbstring php-process php-snmp php-xml php-zip php-mysqlnd python3 python3-PyMySQL python3-redis python3-memcached python3-pip python3-systemd rrdtool unzip
|
||||
```
|
||||
|
||||
=== "Debian 10"
|
||||
=== "NGINX"
|
||||
```
|
||||
apt install acl curl composer fping git graphviz imagemagick mariadb-client mariadb-server mtr-tiny nginx-full nmap php7.3-cli php7.3-curl php7.3-fpm php7.3-gd php7.3-json php7.3-mbstring php7.3-mysql php7.3-snmp php7.3-xml php7.3-zip python3-dotenv python3-pymysql python3-redis python3-setuptools rrdtool snmp snmpd whois
|
||||
apt install acl curl composer fping git graphviz imagemagick mariadb-client mariadb-server mtr-tiny nginx-full nmap php7.3-cli php7.3-curl php7.3-fpm php7.3-gd php7.3-json php7.3-mbstring php7.3-mysql php7.3-snmp php7.3-xml php7.3-zip python3-dotenv python3-pymysql python3-redis python3-setuptools python3-systemd rrdtool snmp snmpd whois
|
||||
```
|
||||
|
||||
## Add librenms user
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
[Unit]
|
||||
Description=LibreNMS SNMP Poller Service
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/librenms/librenms-service.py -v
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
WorkingDirectory=/opt/librenms
|
||||
User=librenms
|
||||
Group=librenms
|
||||
RestartSec=10
|
||||
Restart=always
|
||||
WatchdogSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -2,4 +2,4 @@ PyMySQL
|
|||
python-dotenv
|
||||
redis>=3.0
|
||||
setuptools
|
||||
psutil
|
||||
psutil
|
||||
|
|
|
|||
Loading…
Reference in New Issue