Проект юного DevOps Глава 4: Настройка мониторинга

В других версиях Prometheus процесс и особенности настройки могут отличаться от описанных в данной статье.

В других версиях Node Exporter процесс и особенности настройки могут отличаться от описанных в данной статье.

Приватный адрес сервера Prometheus может отличаться от настроенного в данном стенде.

web.yml opt/node_exporter/
node_exporter usr/bin/
node_exporter.service etc/systemd/system/

Source: node-exporter-lab
Section: unknown
Priority: optional
Maintainer: unknown <nikolay@unknown>
Build-Depends: debhelper-compat (= 13)
Standards-Version: 4.6.0
Homepage: <insert the upstream URL, if relevant>
#Vcs-Browser: https://salsa.debian.org/debian/just-node-exporter
#Vcs-Git: https://salsa.debian.org/debian/just-node-exporter.git
Rules-Requires-Root: no

Package: node-exporter-lab
Architecture: all
Depends: ${misc:Depends}
Description: <insert up to 60 chars description>
 <insert long description, indented with spaces>

#!/bin/sh

set -e

USER="node_exporter"
GROUP="node_exporter"

if ! getent group "$GROUP" > /dev/null 2>&1 ; then
  addgroup --system "$GROUP" --quiet
fi

if ! id "$USER" > /dev/null 2>&1 ; then
  adduser --system --home /usr/share/prometheus --no-create-home \
  --ingroup "$GROUP" --disabled-password --shell /bin/false \
  "$USER"
fi

chown -R $USER:$GROUP /opt/node_exporter
chmod -R 755 /opt/node_exporter
chown $USER:$GROUP /usr/bin/node_exporter
chmod 755 /usr/bin/node_exporter

echo -n "Restarting node_exporter service..."

if command -v systemctl >/dev/null; then
  systemctl daemon-reload || true
  systemctl enable node_exporter.service || true
  systemctl restart node_exporter.service || true
  systemctl restart node_exporter.service || true
fi
echo " OK"

#!/bin/sh

echo 'Stopping and disabling node_exporter service...'

if command -v systemctl >/dev/null; then
  systemctl stop node_exporter.service || true
  systemctl disable node_exporter.service || true
fi

userdel node_exporter >/dev/null 2>&1 || true
groupdel node_exporter >/dev/null 2>&1 || true

echo " OK"

В других версиях OpenVPN Exporter процесс и особенности настройки могут отличаться от описанных в данной статье.

Приватный адрес сервера Prometheus может отличаться от настроенного в данном стенде.

openvpn_exporter usr/bin/
openvpn_exporter.service etc/systemd/system/

Source: openvpn-exporter-lab
Section: unknown
Priority: optional
Maintainer: unknown <nikolay@unknown>
Build-Depends: debhelper-compat (= 13)
Standards-Version: 4.6.0
Homepage: <insert the upstream URL, if relevant>
#Vcs-Browser: https://salsa.debian.org/debian/just-open-vpn-exporter
#Vcs-Git: https://salsa.debian.org/debian/just-open-vpn-exporter.git
Rules-Requires-Root: no

Package: openvpn-exporter-lab
Architecture: all
Depends: ${misc:Depends}
Description: <insert up to 60 chars description>
 <insert long description, indented with spaces>

#!/bin/sh

set -e

USER="openvpn_exporter"
GROUP="openvpn_exporter"
OPENVPN_LOGFILE="/var/log/openvpn/openvpn-status.log"

if ! getent group "$GROUP" > /dev/null 2>&1 ; then
  addgroup --system "$GROUP" --quiet
fi

if ! id "$USER" > /dev/null 2>&1 ; then
  adduser --system --home /usr/share/prometheus --no-create-home \
  --ingroup "$GROUP" --disabled-password --shell /bin/false \
  "$USER"
fi

sudo usermod -a -G openvpn_exporter root

if [ -f "$OPENVPN_LOGFILE" ]; then
  sudo chgrp $GROUP $OPENVPN_LOGFILE
  sudo chmod 660 $OPENVPN_LOGFILE
fi

chown $USER:$GROUP /usr/bin/openvpn_exporter
chmod 755 /usr/bin/openvpn_exporter

echo -n "Restarting openvpn-exporter service..."

if command -v systemctl >/dev/null; then
  systemctl daemon-reload || true
  systemctl enable openvpn_exporter.service || true
  systemctl restart openvpn_exporter.service || true
fi

echo " OK"

#!/bin/sh

USER="openvpn_exporter"
GROUP="openvpn_exporter"
OPENVPN_LOGFILE="/var/log/openvpn/openvpn-status.log"

echo 'Stopping and disabling openvpn-exporter service...'

if command -v systemctl >/dev/null; then
  systemctl stop openvpn_exporter.service || true
  systemctl disable openvpn_exporter.service || true
fi

if [ -f "$OPENVPN_LOGFILE" ]; then
  sudo chgrp root $OPENVPN_LOGFILE
  sudo chmod 640 $OPENVPN_LOGFILE
fi

userdel $USER >/dev/null 2>&1 || true
groupdel $GROUP >/dev/null 2>&1 || true

echo " OK"

В других версиях Nginx Exporter процесс и особенности настройки могут отличаться от описанных в данной статье.

server {
        listen 1111 ssl default_server;
        server_name repo.justnikobird.ru;
        auth_basic              "Restricted Access!";
        auth_basic_user_file    /etc/nginx/conf.d/.htpasswd;
        ssl_certificate     repo.justnikobird.ru.crt;
        ssl_certificate_key repo.justnikobird.ru.key;
        ssl_protocols       TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
        ssl_ciphers         HIGH:!aNULL:!MD5;

        root /var/www/aptly;

        location / {
                autoindex on;
        }
}

# nginx prometheus exporter
server {
        listen 8080;

        location /stub_status {
                stub_status;
                allow 127.0.0.1;
                deny all;
        }
}

Приватный адрес сервера Prometheus может отличаться от настроенного в данном стенде.

nginx-prometheus-exporter usr/bin/
prometheus-nginx-exporter opt/nginx_exporter/
nginx_exporter.service etc/systemd/system/

Source: nginx-exporter-lab
Section: unknown
Priority: optional
Maintainer: Nikolay <justnikobird@yandex.ru>
Build-Depends: debhelper-compat (= 13)
Standards-Version: 4.6.0
Homepage: <insert the upstream URL, if relevant>
#Vcs-Browser: https://salsa.debian.org/debian/just-nginx-exporter
#Vcs-Git: https://salsa.debian.org/debian/just-nginx-exporter.git
Rules-Requires-Root: no

Package: nginx-exporter-lab
Architecture: all
Depends: ${misc:Depends}
Description: <insert up to 60 chars description>
 <insert long description, indented with spaces>

#!/bin/sh

set -e

USER="nginx_exporter"
GROUP="nginx_exporter"

if ! getent group "$GROUP" > /dev/null 2>&1 ; then
  addgroup --system "$GROUP" --quiet
fi

if ! id "$USER" > /dev/null 2>&1 ; then
  adduser --system --home /usr/share/prometheus --no-create-home \
  --ingroup "$GROUP" --disabled-password --shell /bin/false \
  "$USER"
fi

chown -R $USER:$GROUP /opt/nginx_exporter/
chmod -R 755 /opt/nginx_exporter/
chown $USER:$GROUP /usr/bin/nginx-prometheus-exporter
chmod 755 /usr/bin/nginx-prometheus-exporter

echo -n "Restarting nginx_exporter service..."

if command -v systemctl >/dev/null; then
  systemctl daemon-reload || true
  systemctl enable nginx_exporter.service || true
  systemctl restart nginx_exporter.service || true
fi
echo " OK"

#!/bin/sh

echo 'Stopping and disabling nginx-exporter service...'

if command -v systemctl >/dev/null; then
  systemctl stop nginx_exporter.service || true
  systemctl disable nginx_exporter.service || true
fi

userdel nginx_exporter >/dev/null 2>&1 || true
groupdel nginx_exporter >/dev/null 2>&1 || true

echo " OK"

Версия на GitHub

#!/bin/bash

# активируем опцию, которая прерывает выполнение скрипта, если любая команда завершается с ненулевым статусом
set -e

# проверим, запущен ли скрипт от пользователя root
if [[ "${UID}" -ne 0 ]]; then
  echo -e "You need to run this script as root!"
  exit 1
fi

# проверим подключен ли репозиторий
if [[ ! $(grep -rhE ^deb /etc/apt/sources.list*) == *"deb https://repo.justnikobird.ru:1111/lab focal main"* ]]; then
  echo -e "Lab repo not connected!\nPlease run vm_start.sh script!\n"
  exit 1
fi

# функция, которая проверяет наличие пакета в системе и в случае его отсутствия выполняет установку
command_check() {
  if ! command -v "$1" &>/dev/null; then
    echo -e "\n====================\n$2 could not be found!\nInstalling...\n====================\n"
    apt-get install -y "$3"
    echo -e "\nDONE\n"
  fi
}

# функция, которая проверяет наличие правила в iptables и в случае отсутствия применяет его
iptables_add() {
  if ! iptables -C "$@" &>/dev/null; then
    iptables -A "$@"
  fi
}

# функция, которая проверяет корректность введения ip-адреса
ip_request() {
  while true; do
    read -r -p $'\n'"Enter monitor vm ip (format 10.0.0.6): " ip
    if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
      echo "$ip"
      break
    fi
  done
}

# функция, которая проверяет валидность пути в linux-системе
path_request() {
  while true; do
    read -r -e -p $'\n'"Please input valid path to ${1}: " path
    if [ -f "$path" ]; then
      echo "$path"
      break
    fi
  done
}

# установим все необходимые пакеты используя функцию command_check
systemctl restart systemd-timesyncd.service
apt-get update
command_check iptables "Iptables" iptables
command_check netfilter-persistent "Netfilter-persistent" iptables-persistent
command_check basename "Basename" coreutils
command_check htpasswd "Htpasswd" apache2-utils

# выведем в shell меню с предложением выбрать экспортер для установки
while true; do
  echo -e "\n--------------------------\n"
  echo -e "[1] node exporter\n"
  echo -e "[2] openvpn exporter\n"
  echo -e "[3] nginx exporter\n"
  echo -e "[4] exit\n"
  echo -e "--------------------------\n"
  read -r -n 1 -p "Select exporter for install: " exporter

  case $exporter in

  # установим node-exporter
  1)
    echo -e "\n====================\nNode Exporter Installing...\n====================\n"

    # установим ранее собранный пакет node-exporter-lab
    apt-get install -y node-exporter-lab

    # запросим пути до файлов сертификата и ключа
    cert_path=$(path_request certificate)
    key_path=$(path_request key)

    # отделим названия файлов от путей
    cert_file=$(basename "$cert_path")
    key_file=$(basename "$key_path")

    # переместим файлы ключа и сертификата в рабочую директорию программы и поменяем права на владение
    cp "$cert_path" /opt/node_exporter/
    cp "$key_path" /opt/node_exporter/
    chmod 640 /opt/node_exporter/"$cert_file"
    chmod 640 /opt/node_exporter/"$key_file"
    chown node_exporter:node_exporter /opt/node_exporter/"$cert_file"
    chown node_exporter:node_exporter /opt/node_exporter/"$key_file"

    # запросим данные для авторизации и запишем их в конфигурационный файл
    read -r -p $'\n'"Node Exporter username: " username
    read -r -p $'\n'"Node Exporter password: " -s password
    echo -e "tls_server_config:\n  cert_file: $cert_file\n  key_file: $key_file\n\nbasic_auth_users:\n  $username: '$(htpasswd -nbB -C 10 admin "$password" | grep -o "\$.*")'" >/opt/node_exporter/web.yml

    # настроим iptables
    echo -e "\n====================\nIptables configuration\n====================\n"
    monitor_vm_ip=$(ip_request)
    iptables_add INPUT -p tcp -s "$monitor_vm_ip" --dport 9100 -j ACCEPT -m comment --comment prometheus_node_exporter
    echo -e "\n====================\nSaving iptables config\n====================\n"
    service netfilter-persistent save
    echo -e "\nDONE\n"

    # перезагрузим node-exporter-сервис
    systemctl daemon-reload
    systemctl restart node_exporter.service
    systemctl enable node_exporter.service

    echo -e "\n====================\nNode Exporter listening on port 9100\n====================\n"
    echo -e "\nOK\n"
    ;;

  2)
    echo -e "\n====================\nOpenvpn Exporter Installing...\n====================\n"

    # установим ранее собранный пакет openvpn-exporter-lab
    apt-get install -y openvpn-exporter-lab

    # настроим iptables
    echo -e "\n====================\nIptables configuration\n====================\n"
    monitor_vm_ip=$(ip_request)
    iptables_add INPUT -p tcp -s "$monitor_vm_ip" --dport 9176 -j ACCEPT -m comment --comment prometheus_openvpn_exporter
    echo -e "\n====================\nSaving iptables config\n====================\n"
    service netfilter-persistent save
    echo -e "\nDONE\n"

    # перезагрузим openvpn-exporter-сервис
    systemctl daemon-reload
    systemctl restart openvpn_exporter.service
    systemctl enable openvpn_exporter.service

    echo -e "\n====================\nOpenvpn Exporter listening on port 9176\n====================\n"
    echo -e "\nOK\n"
    ;;

  3)
    echo -e "\n====================\nConfigure Nginx /stub_status location on 8080 port before install \n====================\n"
    echo -e "\n====================\nNginx Exporter Installing...\n====================\n"

    # установим ранее собранный пакет nginx-exporter-lab
    apt-get install -y nginx-exporter-lab

    # запросим пути до файлов сертификата и ключа
    cert_path=$(path_request certificate)
    key_path=$(path_request key)

    # отделим названия файлов от путей
    cert_file=$(basename "$cert_path")
    key_file=$(basename "$key_path")

    # переместим файлы ключа и сертификата в рабочую директорию программы и поменяем права на владение
    cp "$cert_path" /opt/nginx_exporter/
    cp "$key_path" /opt/nginx_exporter/
    new_cert_path="/opt/nginx_exporter/$cert_file"
    new_key_path="/opt/nginx_exporter/$key_file"
    chmod 640 "$new_cert_path"
    chmod 640 "$new_key_path"
    chown nginx_exporter:nginx_exporter "$new_cert_path"
    chown nginx_exporter:nginx_exporter "$new_key_path"

    # запишем данные о сертификате и ключе в конфигурационный файл
    echo 'ARGS="-web.secured-metrics -web.ssl-server-cert '"$new_cert_path"' -web.ssl-server-key '"$new_key_path"'' >/opt/nginx_exporter/prometheus-nginx-exporter

    # настроим iptables
    echo -e "\n====================\nIptables configuration\n====================\n"
    monitor_vm_ip=$(ip_request)
    iptables_add INPUT -p tcp -s "$monitor_vm_ip" --dport 9113 -j ACCEPT -m comment --comment prometheus_nginx_exporter
    echo -e "\n====================\nSaving iptables config\n====================\n"
    service netfilter-persistent save
    echo -e "\nDONE\n"

    # перезагрузим nginx-exporter-сервис
    systemctl daemon-reload
    systemctl restart nginx_exporter.service
    systemctl enable nginx_exporter.service

    echo -e "\n====================\nNginx Exporter listening on port 9113\n====================\n"
    echo -e "\nOK\n"
    ;;

  4)
    echo -e "\n\nOK\n"
    exit 0
    ;;

  *)
    echo -e "\n\nUnknown\n"
    ;;
  esac
done

scrape_configs:
  - job_name: 'prometheus-monitor'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: monitor.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['monitor.justnikobird.ru:9090']

  - job_name: 'node-monitor'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: monitor.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['monitor.justnikobird.ru:9100']

  - job_name: 'node-ca'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: ca.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['ca.justnikobird.ru:9100']

  - job_name: 'node-openvpn'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: vpn.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['vpn.justnikobird.ru:9100']

  - job_name: 'node-repo'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: repo.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['repo.justnikobird.ru:9100']

  - job_name: 'openvpn'
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['vpn.justnikobird.ru:9176']

  - job_name: 'nginx-repo'
    scheme: https
    tls_config:
      ca_file: repo.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['repo.justnikobird.ru:9113']

global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
...

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
...

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus-monitor'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: monitor.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['monitor.justnikobird.ru:9090']

  - job_name: 'node-monitor'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: monitor.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['monitor.justnikobird.ru:9100']

  - job_name: 'node-ca'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: ca.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['ca.justnikobird.ru:9100']

  - job_name: 'node-vpn'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: vpn.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['vpn.justnikobird.ru:9100']

  - job_name: 'node-repo'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: repo.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['repo.justnikobird.ru:9100']

  - job_name: 'openvpn'
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['vpn.justnikobird.ru:9176']

  - job_name: 'nginx-repo'
    scheme: https
    tls_config:
      ca_file: repo.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['repo.justnikobird.ru:9113']

amtool: Исполняемый файл, который позволяет просматривать или изменять текущее состояние AlertManager. Другими словами, amtool может отключать оповещения, отключать по истечении срока действия, а также импортировать или запрашивать их. Его можно рассматривать как утилиту для настройки AlertManager без непосредственного изменения конфигурации текущих оповещений.

alertmanager: Исполняемый файл, который используется для запуска сервиса Alert Manager.

alertmanager.yml: Конфигурационный файл AlertManager.

route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 1h
  receiver: 'nikolay'

receivers:
- name: 'nikolay'

  email_configs:
  - to: 'justnikobird@yandex.ru'
    from: 'justnikobird@yandex.ru'
    smarthost: 'smtp.yandex.ru:587'
    auth_username: 'justnikobird'
    auth_identity: 'justnikobird'
    auth_password: '***'

  telegram_configs:
  - bot_token: '6622906782:AAENZl4MCCp0DtzK6XIVqA9q0kp0BtaYC5I'
    chat_id: ***

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
  - scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: monitor.justnikobird.ru.crt
    static_configs:
    - targets: ['monitor.justnikobird.ru:9093']

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - rules.yml

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus-monitor'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: monitor.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['monitor.justnikobird.ru:9090']

  - job_name: 'node-monitor'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: monitor.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['monitor.justnikobird.ru:9100']

  - job_name: 'node-ca'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: ca.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['ca.justnikobird.ru:9100']

  - job_name: 'node-vpn'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: vpn.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['vpn.justnikobird.ru:9100']

  - job_name: 'node-repo'
    scheme: https
    basic_auth:
      username: admin
      password: password
    tls_config:
      ca_file: repo.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['repo.justnikobird.ru:9100']

  - job_name: 'openvpn'
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['vpn.justnikobird.ru:9176']

  - job_name: 'nginx-repo'
    scheme: https
    tls_config:
      ca_file: repo.justnikobird.ru.crt
    scrape_interval: 5s
    scrape_timeout: 5s
    static_configs:
      - targets: ['repo.justnikobird.ru:9113']

groups:
- name: ca
  rules:
  - alert: CA_node_exporter_down
    expr: up{job="node-ca"} == 0
    for: 10s
    annotations:
      title: 'CA Node Exporter Down'
      description: 'CA Node Exporter Down'
    labels:
      severity: 'crit'

  - alert: CA_High_CPU_utiluzation
    expr: node_load1{job="node-ca"} > 0.9
    for: 1m
    annotations:
      title: 'High CPU utiluzation'
      description: 'High CPU utiluzation'
    labels:
      severity: 'crit'

  - alert: CA_High_memory_utiluzation
    expr: ((node_memory_MemAvailable_bytes{job="node-ca"} / node_memory_MemTotal_bytes{job="node-ca"}) * 100) < 10
    for: 1m
    annotations:
      title: 'High memory utiluzation'
      description: 'High memory utiluzation'
    labels:
      severity: 'crit'

  - alert: CA_Disc_space_problem
    expr: ((node_filesystem_avail_bytes{job="node-ca", mountpoint="/",fstype!="rootfs"} / node_filesystem_size_bytes{job="node-ca", mountpoint="/",fstype!="rootfs"}) * 100) < 10
    for: 10m
    annotations:
      title: 'Disk 90% full'
      description: 'Disk 90% full'
    labels:
      severity: 'crit'

  - alert: CA_High_port_incoming_utilization
    expr: (rate(node_network_receive_bytes_total{job="node-ca", device="eth0"}[5m]) / 1024 / 1024) > 150
    for: 5s
    annotations:
      title: 'High port input load'
      description: 'Incoming port load > 150 Mb/s'
    labels:
      severity: 'crit'

  - alert: CA_High_port_outcoming_utilization
    expr: (rate(node_network_transmit_bytes_total{ job="node-ca", device="eth0"}[5m]) / 1024 / 1024) > 150
    for: 5s
    annotations:
      title: High outbound port utilization
      description: 'Outcoming port load > 150 Mb/s'
    labels:
      severity: 'crit'

- name: vpn
  rules:
  - alert: Vpn_node_exporter_down
    expr: up{job="node-vpn"} == 0
    for: 10s
    annotations:
      title: 'Vpn Node Exporter Down'
      description: 'Vpn Node Exporter Down'
    labels:
      severity: 'crit'

  - alert: Vpn_exporter_down
    expr: up{job="openvpn"} == 0
    for: 10s
    annotations:
      title: 'Vpn Exporter Down'
      description: 'Vpn Exporter Down'
    labels:
      severity: 'crit'

  - alert: VpnDown
    expr: openvpn_up == 0
    for: 10s
    annotations:
      title: 'VPN Service down'
      description: 'VPN Service down'
    labels:
      severity: 'crit'

  - alert: Vpn_NoClientConnected
    expr: openvpn_server_connected_clients == 0
    for: 10s
    annotations:
      title: 'No Client Connected'
      description: 'No Client Connected'
    labels:
      severity: 'crit'

  - alert: Vpn_High_CPU_utiluzation
    expr: node_load1{job="node-vpn"} > 0.9
    for: 1m
    annotations:
      title: 'High CPU utiluzation'
      description: 'High CPU utiluzation'
    labels:
      severity: 'crit'

  - alert: Vpn_High_memory_utiluzation
    expr: ((node_memory_MemAvailable_bytes{job="node-vpn"} / node_memory_MemTotal_bytes{job="node-vpn"}) * 100) < 10
    for: 1m
    annotations:
      title: 'High memory utiluzation'
      description: 'High memory utiluzation'
    labels:
      severity: 'crit'

  - alert: Vpn_Disc_space_problem
    expr: ((node_filesystem_avail_bytes{job="node-vpn", mountpoint="/",fstype!="rootfs"} / node_filesystem_size_bytes{job="node-vpn", mountpoint="/",fstype!="rootfs"}) * 100) < 10
    for: 10m
    annotations:
      title: 'Disk 90% full'
      description: 'Disk 90% full'
    labels:
      severity: 'crit'

  - alert: Vpn_High_port_incoming_utilization
    expr: (rate(node_network_receive_bytes_total{job="node-vpn", device="eth0"}[5m]) / 1024 / 1024) > 150
    for: 5s
    annotations:
      title: 'High port input load'
      description: 'Incoming port load > 150 Mb/s'
    labels:
      severity: 'crit'

  - alert: Vpn_High_port_outcoming_utilization
    expr: (rate(node_network_transmit_bytes_total{ job="node-vpn", device="eth0"}[5m]) / 1024 / 1024) > 150
    for: 5s
    annotations:
      title: High outbound port utilization
      description: 'Outcoming port load > 150 Mb/s'
    labels:
      severity: 'crit'

- name: monitor
  rules:
  - alert: Monitor_node_exporter_down
    expr: up{job="node-monitor"} == 0
    for: 10s
    annotations:
      title: 'Monitor Node Exporter Down'
      description: 'Monitor Node Exporter Down'
    labels:
      severity: 'crit'

  - alert: Monitor_prometheus_exporter_down
    expr: up{job="prometheus-monitor"} == 0
    for: 10s
    annotations:
      title: 'Monitor Node Exporter Down'
      description: 'Monitor Node Exporter Down'
    labels:
      severity: 'crit'

  - alert: Monitor_High_CPU_utiluzation
    expr: node_load1{job="node-monitor"} > 0.9
    for: 1m
    annotations:
      title: 'High CPU utiluzation'
      description: 'High CPU utiluzation'
    labels:
      severity: 'crit'

  - alert: Monitor_High_memory_utiluzation
    expr: ((node_memory_MemAvailable_bytes{job="node-monitor"} / node_memory_MemTotal_bytes{job="node-monitor"}) * 100) < 10
    for: 1m
    annotations:
      title: 'High memory utiluzation'
      description: 'High memory utiluzation'
    labels:
      severity: 'crit'

  - alert: Monitor_Disc_space_problem
    expr: ((node_filesystem_avail_bytes{job="node-monitor", mountpoint="/",fstype!="rootfs"} / node_filesystem_size_bytes{job="node-monitor", mountpoint="/",fstype!="rootfs"}) * 100) < 10
    for: 10m
    annotations:
      title: 'Disk 90% full'
      description: 'Disk 90% full'
    labels:
      severity: 'crit'

  - alert: Monitor_High_port_incoming_utilization
    expr: (rate(node_network_receive_bytes_total{job="node-monitor", device="eth0"}[5m]) / 1024 / 1024) > 150
    for: 5s
    annotations:
      title: 'High port input load'
      description: 'Incoming port load > 150 Mb/s'
    labels:
      severity: 'crit'

  - alert: Monitor_High_port_outcoming_utilization
    expr: (rate(node_network_transmit_bytes_total{ job="node-monitor", device="eth0"}[5m]) / 1024 / 1024) > 150
    for: 5s
    annotations:
      title: High outbound port utilization
      description: 'Outcoming port load > 150 Mb/s'
    labels:
      severity: 'crit'

- name: repo
  rules:
  - alert: Repo_node_exporter_down
    expr: up{job="node-repo"} == 0
    for: 10s
    annotations:
      title: 'Repo Node Exporter Down'
      description: 'Repo Node Exporter Down'
    labels:
      severity: 'crit'

  - alert: Repo_nginx_exporter_down
    expr: up{job="nginx-repo"} == 0
    for: 10s
    annotations:
      title: 'Repo Nginx Exporter Down'
      description: 'Repo Nginx Exporter Down'
    labels:
      severity: 'crit'

  - alert: Repo_NginxDown
    expr: nginx_up == 0
    for: 10s
    annotations:
      title: 'Nginx Service down'
      description: 'Nginx Service down'
    labels:
      severity: 'crit'

  - alert: Repo_NoActiveClientConnections
    expr: nginx_connections_active == 1
    for: 10s
    annotations:
      title: 'No active connections'
      description: 'No active connections except nginx exporter'
    labels:
      severity: 'crit'

  - alert: Repo_NoActiveConnections
    expr: nginx_connections_active == 0
    for: 10s
    annotations:
      title: 'No active connections'
      description: 'No active connections'
    labels:
      severity: 'crit'

  - alert: Repo_High_CPU_utiluzation
    expr: node_load1{job="node-repo"} > 0.9
    for: 1m
    annotations:
      title: 'High CPU utiluzation'
      description: 'High CPU utiluzation'
    labels:
      severity: 'crit'

  - alert: Repo_High_memory_utiluzation
    expr: ((node_memory_MemAvailable_bytes{job="node-repo"} / node_memory_MemTotal_bytes{job="node-repo"}) * 100) < 10
    for: 1m
    annotations:
      title: 'High memory utiluzation'
      description: 'High memory utiluzation'
    labels:
      severity: 'crit'

  - alert: Repo_Disc_space_problem
    expr: ((node_filesystem_avail_bytes{job="node-repo", mountpoint="/",fstype!="rootfs"} / node_filesystem_size_bytes{job="node-repo", mountpoint="/",fstype!="rootfs"}) * 100) < 10
    for: 10m
    annotations:
      title: 'Disk 90% full'
      description: 'Disk 90% full'
    labels:
      severity: 'crit'

  - alert: Repo_High_port_incoming_utilization
    expr: (rate(node_network_receive_bytes_total{job="node-repo", device="eth0"}[5m]) / 1024 / 1024) > 150
    for: 5s
    annotations:
      title: 'High port input load'
      description: 'Incoming port load > 150 Mb/s'
    labels:
      severity: 'crit'

  - alert: Repo_High_port_outcoming_utilization
    expr: (rate(node_network_transmit_bytes_total{ job="node-repo", device="eth0"}[5m]) / 1024 / 1024) > 150
    for: 5s
    annotations:
      title: High outbound port utilization
      description: 'Outcoming port load > 150 Mb/s'
    labels:
      severity: 'crit'

alertmanager.yml etc/prometheus
console_libraries/ etc/prometheusi
consoles/ etc/prometheus
prometheus.yml etc/prometheus
rules.yml etc/prometheus
web.yml etc/prometheus
prometheus usr/bin
promtool usr/bin
alertmanager usr/bin
amtool usr/bin
prometheus.service etc/systemd/system
prometheus-alertmanager.service etc/systemd/system

Source: prometheus-lab
Section: unknown
Priority: optional
Maintainer: Nikolay <justnikobird@yandex.ru>
Build-Depends: debhelper-compat (= 13)
Standards-Version: 4.6.0
Homepage: <insert the upstream URL, if relevant>
#Vcs-Browser: https://salsa.debian.org/debian/just-prometheus
#Vcs-Git: https://salsa.debian.org/debian/just-prometheus.git
Rules-Requires-Root: no

Package: prometheus-lab
Architecture: all
Depends: ${misc:Depends}
Description: <insert up to 60 chars description>
 <insert long description, indented with spaces>

#!/bin/sh

set -e

USER="prometheus"
GROUP="prometheus"

if ! getent group "$GROUP" > /dev/null 2>&1 ; then
  addgroup --system "$GROUP" --quiet
fi

if ! id "$USER" > /dev/null 2>&1 ; then
  adduser --system --home /usr/share/prometheus --no-create-home \
  --ingroup "$GROUP" --disabled-password --shell /bin/false \
  "$USER"
fi

if [ ! -d "/var/lib/prometheus/" ]; then
  mkdir /var/lib/prometheus/
  chown -R $USER:$GROUP /var/lib/prometheus/
  chmod -R 755 /var/lib/prometheus/
fi

if [ ! -d "/etc/prometheus/alertmanager_data/" ]; then
  mkdir /etc/prometheus/alertmanager_data/
  chown -R $USER:$GROUP /etc/prometheus/alertmanager_data/
  chmod -R 755 /etc/prometheus/alertmanager_data/
fi

chown -R $USER:$GROUP /etc/prometheus/
chmod -R 755 /etc/prometheus/
chown $USER:$GROUP /usr/bin/prometheus
chmod 755 /usr/bin/prometheus
chown $USER:$GROUP /usr/bin/promtool
chmod 755 /usr/bin/promtool
chown $USER:$GROUP /usr/bin/alertmanager
chmod 755 /usr/bin/alertmanager
chown $USER:$GROUP /usr/bin/amtool
chmod 755 /usr/bin/amtool

ln -s /usr/bin/prometheus /usr/local/bin/prometheus
chown -h prometheus:prometheus /usr/local/bin/prometheus

ln -s /usr/bin/promtool /usr/local/bin/promtool
chown -h prometheus:prometheus /usr/local/bin/promtool

ln -s /usr/bin/alertmanager /usr/local/bin/alertmanager
chown -h prometheus:prometheus /usr/local/bin/alertmanager

ln -s /usr/bin/amtool /usr/local/bin/amtool
chown -h prometheus:prometheus /usr/local/bin/amtool

echo -n "Restarting prometheus-server service..."

if command -v systemctl >/dev/null; then
  systemctl daemon-reload
  systemctl restart prometheus.service || true
  systemctl restart prometheus-alertmanager.service || true
fi

echo " OK"

#!/bin/sh

echo 'Stopping and disabling prometheus-server service...'

if command -v systemctl >/dev/null; then
  systemctl stop prometheus.service || true
  systemctl disable prometheus.service || true
  systemctl stop prometheus-alertmanager.service || true
  systemctl disable prometheus-alertmanager.service || true
fi

userdel prometheus >/dev/null 2>&1 || true
groupdel prometheus >/dev/null 2>&1 || true

if [ -d "/var/lib/prometheus/" ]; then
  rm -rf /var/lib/prometheus/
fi

if [ -d "/etc/prometheus/alertmanager_data/" ]; then
  rm -rf /etc/prometheus/alertmanager_data/
fi

unlink /usr/local/bin/prometheus 2>&1 || true
unlink /usr/local/bin/promtool 2>&1 || true
unlink /usr/local/bin/alertmanager 2>&1 || true
unlink /usr/local/bin/amtool 2>&1 || true

echo " OK"

Версия на GitHub

#!/bin/bash

# активируем опцию, которая прерывает выполнение скрипта, если любая команда завершается с ненулевым статусом
set -e

# проверим, запущен ли скрипт от пользователя root
if [[ "${UID}" -ne 0 ]]; then
  echo -e "You need to run this script as root!"
  exit 1
fi

# проверим подключен ли репозиторий
if [[ ! $(grep -rhE ^deb /etc/apt/sources.list*) == *"deb https://repo.justnikobird.ru:1111/lab focal main"* ]]; then
  echo -e "Lab repo not connected!\nPlease run vm_start.sh script!\n"
  exit 1
fi

# функция, которая проверяет наличие пакета в системе и в случае его отсутствия выполняет установку
command_check() {
  if ! command -v "$1" &>/dev/null; then
    echo -e "\n====================\n$2 could not be found!\nInstalling...\n====================\n"
    apt-get install -y "$3"
    echo -e "\nDONE\n"
  fi
}

# функция, которая проверяет наличие правила в iptables и в случае отсутствия применяет его
iptables_add() {
  if ! iptables -C "$@" &>/dev/null; then
    iptables -A "$@"
  fi
}

# функция, которая проверяет валидность пути в linux-системе
path_request() {
  while true; do
    read -r -e -p $'\n'"Please input valid path to ${1}: " path
    if [ -f "$path" ]; then
      echo "$path"
      break
    fi
  done
}

# установим все необходимые пакеты используя функцию command_check
systemctl restart systemd-timesyncd.service
apt-get update
command_check iptables "Iptables" iptables
command_check netfilter-persistent "Netfilter-persistent" iptables-persistent
command_check prometheus "Prometheus" prometheus-lab
command_check basename "Basename" coreutils
command_check htpasswd "Htpasswd" apache2-utils

# запросим адрес приватной сети и проверим его на корректность
while true; do
  read -r -p $'\n'"Privat network (format 10.0.0.0/24): " private_net
  if [[ ! $private_net =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/[0-9]{1,2}$ ]]; then
    echo -e "\nPrefix not valid!\n"
  else
    break
  fi
done

# выполним настройку iptables
echo -e "\n====================\nIptables configuration\n====================\n"
iptables_add INPUT -p tcp --dport 9090 -j ACCEPT -m comment --comment prometheus
iptables_add INPUT -p tcp --dport 9093 -j ACCEPT -m comment --comment prometheus_alertmanager
iptables_add OUTPUT -p tcp --dport 587 -j ACCEPT -m comment --comment smtp
iptables_add OUTPUT -p tcp -d "$private_net" --dport 9100 -j ACCEPT -m comment --comment prometheus_node_exporter
iptables_add OUTPUT -p tcp -d "$private_net" --dport 9176 -j ACCEPT -m comment --comment prometheus_openvpn_exporter
iptables_add OUTPUT -p tcp -d "$private_net" --dport 9113 -j ACCEPT -m comment --comment prometheus_nginx_exporter
echo -e "\n====================\nSaving iptables config\n====================\n"
service netfilter-persistent save
echo -e "\nDONE\n"

# выполним настройку HTTPS
echo -e "\n====================\nHTTPS configuration\n====================\n"

# запросим путь до файла сертификата, перенесем его в рабочую директорию программы и поменяем владельца
cert_path=$(path_request certificate)
cp "$cert_path" /etc/prometheus/
cert_file=$(basename "$cert_path")
chmod 640 /etc/prometheus/"$cert_file"
chown prometheus:prometheus /etc/prometheus/"$cert_file"

# запросим путь до файла ключа, перенесем его в рабочую директорию программы и поменяем владельца
key_path=$(path_request key)
cp "$key_path" /etc/prometheus/
key_file=$(basename "$key_path")
chmod 640 /etc/prometheus/"$key_file"
chown prometheus:prometheus /etc/prometheus/"$key_file"

# перенесем в директорию prometheus сертификаты экспортеров
while true; do
  read -r -n 1 -p $'\n\n'"Add exporter's certificate to prometheus directory? (y|n) " yn
  case $yn in
  [Yy]*)
    echo -e "\n"
    exp_cert_path=$(path_request certificate)
    cp "$exp_cert_path" /etc/prometheus/
    exp_cert_file=$(basename "$exp_cert_path")
    chmod 640 /etc/prometheus/"$exp_cert_file"
    chown prometheus:prometheus /etc/prometheus/"$exp_cert_file"
    ;;

  [Nn]*)
    echo -e "\n"
    break
    ;;

  *) echo -e "\nPlease answer Y or N!\n" ;;
  esac
done

# запросим username и password для авторизации в программе
read -r -p $'\n'"Prometheus username: " username
read -r -p $'\n'"Prometheus password: " -s password

# запросим доменное имя для подключения prometheus к alertmanager
read -r -p $'\n\n'"Prometheus domain name (format monitor.justnikobird.ru): " domain_name

# запишем настройки в конфигурационный файл /etc/prometheus/web.yml
echo -e "tls_server_config:\n  cert_file: $cert_file\n  key_file: $key_file\n\nbasic_auth_users:\n  $username: '$(htpasswd -nbB -C 10 admin "$password" | grep -o "\$.*")'" >/etc/prometheus/web.yml

# внесем изменения в конфигурационный файл /etc/prometheus/prometheus.yml в блок alerting
sed -r -i '0,/(^.*\susername:\s).*$/s//\1'"$username"'/' /etc/prometheus/prometheus.yml
sed -r -i '0,/(^.*\spassword:\s).*$/s//\1'"$password"'/' /etc/prometheus/prometheus.yml
sed -r -i '0,/(^.*\sca_file:\s).*$/s//\1'"$cert_file"'/' /etc/prometheus/prometheus.yml
sed -r -i "0,/(^.*\stargets:\s).*/s//\1['$domain_name:9093']/" /etc/prometheus/prometheus.yml

# выполним настройку DNS
echo -e "\n\n====================\nDNS configuration\n====================\n"

# закрепим доменное имя prometheus за адресом localhost
if ! grep -Fxq "127.0.0.1 $domain_name" /etc/hosts &>/dev/null; then
  echo "127.0.0.1 $domain_name" >>/etc/hosts
  echo -e "\nString '127.0.0.1 $domain_name' added to /etc/hosts\n\n"
fi

echo -e "/etc/hosts file content:\n\n"
cat /etc/hosts

# запросим у пользователя строки для добавления в /etc/hosts
while true; do
  read -r -n 1 -p $'\n\n'"Add new string to /etc/hosts? (y|n) " yn
  case $yn in
  [Yy]*)
    while true; do
      read -r -p $'\n\n'"Enter string in format '<ip> <domain>': " domain_str
      if [[ $domain_str =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}[[:blank:]][a-z\.]+$ ]]; then
        echo "$domain_str" >>/etc/hosts
        echo -e "\n\n/etc/hosts file content:\n\n"
        cat /etc/hosts
        break
      else
        echo -e "\nWrong string format!\n"
      fi
    done
    ;;

  [Nn]*)
    echo -e "\n"
    break
    ;;

  *) echo -e "\nPlease answer Y or N!\n" ;;
  esac
done

# перезагрузим сервисы prometheus и alertmanager
echo -e "\nDONE\n"
systemctl daemon-reload
systemctl restart prometheus.service
systemctl enable prometheus.service
systemctl restart prometheus-alertmanager.service
systemctl enable prometheus-alertmanager.service

echo -e "\n====================\nPrometheus listening on port 9090\nAlertmanager listening on port 9093\n====================\n"
echo -e "\nOK\n"
exit 0

В других версиях Grafana процесс и особенности настройки могут отличаться от описанных в данной статье.

Версия на GitHub

#!/bin/bash

# активируем опцию, которая прерывает выполнение скрипта, если любая команда завершается с ненулевым статусом
set -e

# проверим, запущен ли скрипт от пользователя root
if [[ "${UID}" -ne 0 ]]; then
  echo -e "You need to run this script as root!"
  exit 1
fi

# проверим подключен ли репозиторий
if [[ ! $(grep -rhE ^deb /etc/apt/sources.list*) == *"deb https://repo.justnikobird.ru:1111/lab focal main"* ]]; then
  echo -e "Lab repo not connected!\nPlease run vm_start.sh script!\n"
  exit 1
fi

# функция, которая проверяет наличие пакета в системе и в случае его отсутствия выполняет установку
command_check() {
  if ! command -v "$1" &>/dev/null; then
    echo -e "\n====================\n$2 could not be found!\nInstalling...\n====================\n"
    apt-get install -y "$3"
    echo -e "\nDONE\n"
  fi
}

# функция, которая проверяет наличие правила в iptables и в случае отсутствия применяет его
iptables_add() {
  if ! iptables -C "$@" &>/dev/null; then
    iptables -A "$@"
  fi
}

# функция, которая проверяет валидность пути в linux-системе
path_request() {
  while true; do
    read -r -e -p $'\n'"Please input valid path to ${1}: " path
    if [ -f "$path" ]; then
      echo "$path"
      break
    fi
  done
}

# установим все необходимые пакеты используя функцию command_check
systemctl restart systemd-timesyncd.service
apt-get update
command_check iptables "Iptables" iptables
command_check netfilter-persistent "Netfilter-persistent" iptables-persistent
command_check basename "Basename" coreutils
command_check grafana-server "Grafana" grafana-lab

# настроим iptables
echo -e "\n====================\nIptables configuration\n====================\n"
iptables_add INPUT -p tcp --dport 3000 -j ACCEPT -m comment --comment grafana
echo -e "\n====================\nSaving iptables config\n====================\n"
service netfilter-persistent save
echo -e "\nDONE\n"

# настроим https
echo -e "\n====================\nHTTPS configuration\n====================\n"

# запросим путь до сертификата и перенесем его в рабочую директорию программы
cert_path=$(path_request certificate)
cp "$cert_path" /etc/grafana/
# отделим название файла от пути и поменяем права на доступ
cert_file=$(basename "$cert_path")
chmod 640 /etc/grafana/"$cert_file"
chown root:grafana /etc/grafana/"$cert_file"

# запросим путь до ключа и перенесем его в рабочую директорию программы
key_path=$(path_request key)
cp "$key_path" /etc/grafana/
# отделим название файла от пути и поменяем права на доступ
key_file=$(basename "$key_path")
chmod 640 /etc/grafana/"$key_file"
chown root:grafana /etc/grafana/"$key_file"

# запросим доменное имя
read -r -e -p $'\n'"Please input domain name (example: monitor.justnikobird.ru): " domain

# внесем настройки в конфигурационный файл grafana
sed -i 's/^\;\(protocol\).*$/\1 = https/' /etc/grafana/grafana.ini
sed -i 's/^\;\(domain\).*$/\1 = '"$domain"'/' /etc/grafana/grafana.ini
sed -i 's@^\;\(cert_file\).*$@\1 = /etc/grafana/'"$cert_file"'@' /etc/grafana/grafana.ini
sed -i 's@^\;\(cert_key\).*$@\1 = /etc/grafana/'"$key_file"'@' /etc/grafana/grafana.ini

echo -e "\nDONE\n"

# перезагрузим сервис grafana
systemctl daemon-reload
systemctl restart grafana-server.service
systemctl enable grafana-server.service

echo -e "\n====================\nGrafana listening on port 3000\n====================\n"
echo -e "\nOK\n"
exit 0