2. Prometheus setup

2.1 Install Prometheus

2.1.1 Create group and user for Prometheus:

root@deploymentmachine:/home# id
uid=0(root) gid=0(root) groups=0(root)
root@deploymentmachine:/home# groupadd --system prometheus
root@deploymentmachine:/home# useradd -s /sbin/nologin --system -g prometheus prometheus

2.1.2 Create directories for Prometheus:

root@deploymentmachine:/home# id
uid=0(root) gid=0(root) groups=0(root)
root@deploymentmachine:/home#  mkdir -p /var/lib/prometheus
root@deploymentmachine:/home# mkdir -p /etc/prometheus/
root@deploymentmachine:/home# cd /etc/prometheus/
root@deploymentmachine:/etc/prometheus# mkdir {rules,rules.d}

2.1.3 Download the latest version from the official repository:

Releases · prometheus/prometheusGitHub

root@deploymentmachine:/home# curl -s https://github.com/prometheus/prometheus/releases | grep latest
<div class="release pt-2 pt-md-0 pb-3 pb-md-0 clearfix label-latest">
            <a class="border-0 Label--outline-green" href="/prometheus/prometheus/releases/latest">Latest release</a>
              <a class="border-0 Label--outline-green" href="/prometheus/prometheus/releases/latest">Latest release</a>
root@deploymentmachine:/home#

better use api.github.com endpoints...

root@deploymentmachine:/home# curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest |\
grep browser_download_url | \
grep linux-amd64.tar.gz | \
awk '{gsub(/"/, "", $2); print $2}' | wget -qi -
root@deploymentmachine:/home#
root@deploymentmachine:/home# ls -ltr prom*
-rw-r--r-- 1 root root 65509305 Jan 20 02:12 prometheus-2.24.1.linux-amd64.tar.gz

... and untar for extracting files:

root@deploymentmachine:/home# tar xvf prom*.tar.gz
root@deploymentmachine:/home# ls prom*
prometheus-2.24.1.linux-amd64.tar.gz

prometheus-2.24.1.linux-amd64:
LICENSE  NOTICE  console_libraries  consoles  prometheus  prometheus.yml  promtool
root@deploymentmachine:/home#

2.4 Move the following files under /usr/local/bin, respectively /etc/prometheus

root@deploymentmachine:/home# cd prom*/
root@deploymentmachine:/home/prometheus-2.24.1.linux-amd64# mv prometheus promtool /usr/local/bin/
root@deploymentmachine:/home/prometheus-2.24.1.linux-amd64# mv prometheus.yml  /etc/prometheus/prometheus.yml
root@deploymentmachine:/home/prometheus-2.24.1.linux-amd64# mv consoles/ console_libraries/ /etc/prometheus/

2.2. Configure Prometheus

2.2.1 Create a systemd file for Prometheus service, with following content:

root@deploymentmachine:/home# cat /etc/systemd/system/prometheus.service
[Unit]
 Description=Prometheus
 Documentation=https://prometheus.io/docs/introduction/overview/
 Wants=network-online.target
 After=network-online.target

 [Service]
 Type=simple
 Environment="GOMAXPROCS=1"
 User=prometheus
 Group=prometheus
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/prometheus \
 --config.file=/etc/prometheus/prometheus.yml \
 --storage.tsdb.path=/var/lib/prometheus \
 --web.console.templates=/etc/prometheus/consoles \
 --web.console.libraries=/etc/prometheus/console_libraries \
 --web.listen-address=0.0.0.0:9090 \
 --web.external-url=

 SyslogIdentifier=prometheus
 Restart=always

 [Install]
 WantedBy=multi-user.target

The configuration file prometheus.yml that has been moved under /etc/prometheus/prometheus.yaml should have the following content:

root@deploymentmachine:/home# more /etc/prometheus/prometheus.yml

******** prometheus: Not a text file ********

root@deploymentmachine:/home/tests/mydb/chew/prometheus-2.24.1.linux-amd64# more prometheus.yml
# my global config
global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
    - targets: ['localhost:9090']

We will use this file for addition of targets and job_names for our MySQL services

2.2.2 Change ownership and permission of folder /etc/prometheus:

root@deploymentmachine:/home# chown -R prometheus:prometheus /etc/prometheus/
root@deploymentmachine:/home# chmod -R 775 /etc/prometheus/

2.2.3. Reload and start the Prometheus service:

root@deploymentmachine:/home# systemctl daemon-reload
root@deploymentmachine:/home# systemctl enable prometheus
root@deploymentmachine:/home# systemctl start prometheus

2.2.4 Perform checking

a) Check if the port 9090 is listening:

root@deploymentmachine:/home# lsof -i :9090
COMMAND     PID       USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
prometheu 20617 prometheus    8u  IPv6 235347      0t0  TCP *:9090 (LISTEN)

b) Check the status of the service

root@deploymentmachine:/home# systemctl status prometheus.service  | grep active
     Active: active (running) since Tue 2021-02-02 15:38:57 UTC; 2h 20min ago

c) Open a browser of your choice, and see if you can reach Prometheus at Public IP of your Cloud instance, and port 9090: