构建企业级监控平台系列（二十四）：Prometheus 配置 Grafana 展示与报警

Prometheus服务器 192.168.109.138   Prometheus、node_exporter
grafana服务器     192.168.109.138   Grafana
被监控服务器       192.168.109.0/24  node_exporter

#解压上传后的软件包
root@localhost opt]# tar xf prometheus-2.35.0.linux-amd64.tar.gz
#移动并命名
[root@localhost opt]# mv prometheus-2.35.0.linux-amd64 /usr/local/prometheus
[root@localhost opt]# cd /usr/local/prometheus
[root@localhost prometheus]# ls
console_libraries  consoles  LICENSE  NOTICE  prometheus  prometheus.yml  promtool

cat /usr/local/prometheus/prometheus.yml | grep -v "^#"
global:     #用于prometheus的全局配置，比如采集间隔，抓取超时时间等
  scrape_interval: 15s   #采集目标主机监控数据的时间间隔，默认为1m
  evaluation_interval: 15s   #触发告警生成alert的时间间隔，默认是1m
  # scrape_timeout is set to the global default (10s).
  scrape_timeout: 10s   #数据采集超时时间，默认10s

alerting:    #用于alertmanager实例的配置，支持静态配置和动态服务发现的机制
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

rule_files:    #用于加载告警规则相关的文件路径的配置，可以使用文件名通配机制
  # - "first_rules.yml"
  # - "second_rules.yml"

scrape_configs:   #用于采集时序数据源的配置
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"  #每个被监控实例的集合用job_name命名，支持静态配置（static_configs）和动态服务发现的机制（*_sd_configs）

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:    #静态目标配置，固定从某个target拉取数据
      - targets: ["localhost:9090"]

cat > /usr/lib/systemd/system/prometheus.service <<'EOF'
[Unit]
Description=Prometheus Server
Documentation=https://prometheus.io
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/prometheus/prometheus \
--config.file=/usr/local/prometheus/prometheus.yml \
--storage.tsdb.path=/usr/local/prometheus/data/ \
--storage.tsdb.retention=15d \
--web.enable-lifecycle
  
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

---------------------------------------------------------------
[Unit]  #服务单元
Description=Prometheus Server  #描述
Documentation=https://prometheus.io  
After=network.target   #依赖关系

[Service]
Type=simple
ExecStart=/usr/local/prometheus/prometheus \
--config.file=/usr/local/prometheus/prometheus.yml \  #配置文件
--storage.tsdb.path=/usr/local/prometheus/data/ \  #数据目录
--storage.tsdb.retention=15d \  #保存时间
--web.enable-lifecycle  #开启热加载
  
ExecReload=/bin/kill -HUP $MAINPID  #重载
Restart=on-failure

[Install]
WantedBy=multi-user.target

systemctl start prometheus
systemctl enable prometheus

netstat -natp | grep :9090

cd /opt/
tar xf node_exporter-1.3.1.linux-amd64.tar.gz
mv node_exporter-1.3.1.linux-amd64/node_exporter /usr/local/bin

cat > /usr/lib/systemd/system/node_exporter.service <<'EOF'
[Unit]
Description=node_exporter
Documentation=https://prometheus.io/
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/node_exporter \
--collector.ntp \
--collector.mountstats \
--collector.systemd \
--collector.tcpstat

ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl start node_exporter
systemctl enable node_exporter

netstat -natp | grep :9100

vim /usr/local/prometheus/prometheus.yml
#在尾部增加如下内容
  - job_name: nodes
    metrics_path: "/metrics"
    static_configs:
    - targets:
   - 192.168.109.138:9100
   - 192.168.109.137:9100
   - 192.168.109.136:9100
      labels:
        service: kubernetes

curl -X POST http://192.168.109.138:9090/-/reload     #热加载
或systemctl reload prometheus

#使用yum解决依赖关系  我这边直接上传软件包到opt
yum install -y grafana-7.4.0-1.x86_64.rpm

systemctl start grafana-server
systemctl enable grafana-server

netstat -natp | grep :3000

cd /usr/local/prometheus
mkdir targets

vim targets/node-exporter.yaml
- targets:
  - 192.168.109.131:9100
  - 192.168.109.132:9100
  - 192.168.109.133:9100
  labels:
    app: node-exporter
    job: node

#修改 prometheus 配置文件，发现 target 的配置，定义在配置文件的 job 之中
vim /usr/local/prometheus/prometheus.yml
......
scrape_configs:
  - job_name: nodes
    file_sd_configs:                  #指定使用文件服务发现
    - files:                          #指定要加载的文件列表
      - targets/node*.yaml            #文件加载支持通配符
      refresh_interval: 2m            #每隔 2 分钟重新加载一次文件中定义的 Targets，默认为 5m
systemctl reload prometheus
浏览器查看 Prometheus 页面的 Status -> Targets

cd /opt/
unzip consul_1.9.2_linux_amd64.zip
mv consul /usr/local/bin/

#创建 Consul 服务的数据目录和配置目录
mkdir /var/lib/consul-data
mkdir /etc/consul/

#使用 server 模式启动 Consul 服务
consul agent \
-server \
-bootstrap \
-ui \
-data-dir=/var/lib/consul-data \
-config-dir=/etc/consul/ \
-bind=192.168.109.138 \
-client=0.0.0.0 \
-node=consul-server01 &> /var/log/consul.log &

#查看 consul 集群成员
consul members

#在配置目录中添加文件
vim /etc/consul/nodes.json
{
  "services": [
    {
      "id": "node_exporter-node01",
      "name": "node01",
      "address": "192.168.109.138",
      "port": 9100,
      "tags": ["nodes"],
      "checks": [{
        "http": "http://192.168.109.138:9100/metrics",
        "interval": "5s"
      }]
    },
    {
      "id": "node_exporter-node02",
      "name": "node02",
      "address": "192.168.109.134",
      "port": 9100,
      "tags": ["nodes"],
      "checks": [{
        "http": "http://192.168.109.134:9100/metrics",
        "interval": "5s"
      }]
    }
  ]
}

#让 consul 重新加载配置信息
consul reload  

vim /usr/local/prometheus/prometheus.yml
......
  - job_name: nodes
    consul_sd_configs:                  #指定使用 consul 服务发现
    - server: 192.168.109.138:8500        #指定 consul 服务的端点列表
      tags:                             #指定 consul 服务发现的 services 中哪些 service 能够加入到 prometheus 监控的标签
      - nodes
      refresh_interval: 2m

systemctl reload prometheus

#让 consul 注销 Service
consul services deregister -id="node_exporter-node02"

#重新注册
consul services register /etc/consul/nodes.json