6.Prometheus监控入门之企业监控实战警报发送

alerting:
  alertmanagers:
  - follow_redirects: true
    scheme: http
    timeout: 10s
    api_version: v2
    static_configs:
    - targets:
      - localhost:9093

# 个人理解(伪配置)
group_by: ['team']
  group_by: ['env','region']
  match_re: 
    severity: (page|ticket) # 重要事件&通知单

# severity标签值为ticket和page的记录也将按照region和env进行分组。
        env {page|ticket} 
team -> 
        region (page|ticket)

# (1) 全局设置
global:
  # - QQ企业邮件通知相关配置
  resolve_timeout: 5m
  smtp_from: 'monitor@weiyigeek.top'
  smtp_smarthost: 'smtp.exmail.qq.com:465'
  smtp_auth_username: 'monitor@weiyigeek.top'
  smtp_auth_password: 'xxxxxxxxxxxxxxx'
  smtp_require_tls: false

# (2) 告警模板&告警接收者
templates:
- '/etc/alertmanager/*.tmpl'

- name: 'email-notify'
  email_configs:
  - send_resolved: true  # 如设置为true问题已解决将发送信息,如设置false则不发送
    from: 'monitor@weiyigeek.top'
    to: '{{ template "email.to" . }}'
    html: '{{ template "email.to.html" . }}'
   
- name: 'wechat-notify'
  wechat_configs:
  - send_resolved: true
    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'
    corp_id: 'ww32cf0bf4e14b11487'
    agent_id: '1000002'
    api_secret: 'SYCdXbWj5SjdT9p1OSQnjIZ4RF_SIRqERVuZNWE1yE5g'
    to_user: '@all'
    to_party: '1'
    message: '{{ template "wechat.default.message" . }}'

- name: 'wehook-notify'
  webhook_configs:
  - url: http://webhook.weiyigeek.top:8888/alert


# 抑制告警规则: 如果存在severity标签值为critical告警，将会抑制带有severity标签值为warning以及service标签的警告。
inhibit_rules:
- source_match:
    severity: 'critical'
  target_match:
    severity: 'warning'
  equal: ['service']

# (3) 路由树
route:
  # 路由默认通知接收器为邮箱通知
  receiver: 'email-notify'  
  group_by: ['team'] # 默认路由告警按照team标签进行分组，如没有该标签的告警将不会进行后续匹配。
  group_wait: 30s    # 当传入警报创建新的警报组时，请至少等待“group_wait”以发送初始通知。
  group_interval: 5m  # 发送第一个通知时，请等待“group_interval”以发送一批新警报，这些警报已开始为该组触发。
  repeat_interval: 4h # 如果警报已成功发送，请等待“重复间隔”发送警报。
  # continue: true    # 当匹配一个路由后它不会停止继续向下查找，主要用于将告警记录记录到另外一个系统。
  # 非常注意: 所有上述属性都由所有子路由继承，并且可以在每个子路由上进行覆盖。

  # * 子路由设置 
  routes:
  # 前端团队，开发测试环境主机(team/severity)分别是在告警规则文件中设置的标签。
  - match:
      team: frontend
      env: dev
    group_by: ['region','env'] # frontend 团队根据region和env标签对告警进行分组，它针对后续子路有效，因此它们severity标签值为ticket和page的记录也将按照region和env进行分组。
    group_interval: 6m       # 组警告之间的间隔时间
    receiver: 'email-notify' # 没有匹配到下述条件的告警将发送给该接收者。
    # 还可嵌套子路由
    routers:
    - match: 
        severity: page  # 重要事件
      receiver: 'frontend-pager'
      group_wait: 1m        # 组警告等待
    - match: 
        severity: ticket # 通知单
      receiver: 'frontend-pager'

  # 后端团队，开发测试环境主机
  - match:
      team: backend
      env: dev
    receiver: 'email-notify'
    # 还可嵌套子路由
    routers:
    - match: 
        severity: page  # 重要事件
      receiver: 'backend-pager'
    - match: 
        severity: ticket # 通知单
      receiver: 'backend-pager'

  # 全团队，线上环境
  - match:
      env: prod
    receiver: 'wechat-notify'
    # 还可嵌套子路由
    routers:
    - match: 
        severity: critical     # 极为严重
      receiver: 'wechat-notify-manager'
    # 支持正则匹配就不用写多行标签了。
    - match_re: 
        severity: (page|ticket) # 重要事件&通知单
      receiver: 'wechat-notify-devops'

groups:
- name: node-alert
  rules:
  - alert: "监控节点丢失或宕机"
    expr: avg without()(up) < 1
    for: 2m
    labels:
      severity: 'critical'
    annotations:
      message: "服务器实例 [{{ $labels.instance }}]节点丢失或宕机,"
      description: "请系统管理员尽快进行人工干预处理！"
  - alert: "L-五分钟内CPU使用率大于90%"
    expr: (node_load5 /ignoring(mode) count without(cpu)(node_cpu_seconds_total{mode="idle"}) * 100) > 90
    for: 3m
    labels:
      severity: warning
    annotations:
      message: "服务器实例 {{ $labels.instance }} CPU 使用率 告警通知"
      description: "{{ $labels.instance }}CPU使用率已超过90%, 当前值: {{ $value }}%"

  - alert: "L-内存可用容量小于 10%"
    expr: ((node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Buffers_bytes - node_memory_Cached_bytes) / (node_memory_MemTotal_bytes )) * 100 > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      message: "服务器实例 {{ $labels.instance }} 内存不足 告警通知"
      description: "{{ $labels.instance }} 内存可用资源已不足 10%,当前值: {{ $value }} %"

  - alert: "L-磁盘可用容量小于 10%"
    expr: round(100 - ((node_filesystem_avail_bytes{mountpoint=~".*",fstype=~"nfs|ext4|xfs|ext2|ext3"} * 100) / node_filesystem_size_bytes {mountpoint=~".*",fstype=~"nfs|ext4|xfs|ext2|ext3"}) > 90)
    for: 3m
    labels:
      severity: warning
    annotations:
      message: "服务器实例 {{ $labels.instance }} 磁盘不足 告警通知"
      description: "{{ $labels.instance }} 磁盘 {{ $labels.device }} 资源 已不足 10%, 当前值: {{ $value }}%"

  - alert: "W-内存可用容量小于 10%"
    expr: round((1 - (windows_os_physical_memory_free_bytes / windows_cs_physical_memory_bytes)) * 100) > 90
    for: 2m
    labels:
      severity: warning
    annotations:
      message: "服务器实例 {{ $labels.instance }} 内存不足 告警通知"
      description: "{{ $labels.instance }} 内存可用资源已不足 10%,当前值: {{ $value }}"

  - alert: "W-磁盘可用容量小于 10%"
    expr: round((1 - windows_logical_disk_free_bytes{volume=~"C:|D:|E:|F:"} / windows_logical_disk_size_bytes{volume=~"C:|D:|E:|F:"} ) * 100 > 90)
    for: 3m
    labels:
      severity: warning
    annotations:
      message: "服务器实例 {{ $labels.instance }} 磁盘不足 告警通知"
      description: "{{ $labels.instance }} 磁盘 {{ $labels.volume }} 资源 已不足 10%, 当前值: {{ $value }}%"

  - alert: "W-五分钟内CPU使用率大于 90%"
    expr: round(100 - ((avg without(core,mode)(irate(windows_cpu_time_total{mode="idle"}[5m]))) * 100) > 90)
    for: 3m
    labels:
      severity: warning
    annotations:
      message: "服务器实例 {{ $labels.instance }} CPU 使用率 告警通知"
      description: "{{ $labels.instance }}CPU使用率已超过90%, 当前值: {{ $value }}%"

  - alert: "Website-网站服务访问状态异常"
    expr: probe_success{job="HTTP-Check"} < 1
    for: 5m
    labels:
      severity: warning
    annotations:
      message: "网站地址 {{ $labels.http }} ，无法正常提供服务"
      description: "请检查并恢复网站名称为 {{ $labels.site }} 应用服务，当前值: {{ $value }}%"

# 模板代码块必须使用 {{ var }} 包含

# 1.定义包含
{{ define "wechat.custom.message" }} 
  app: {{ .GroupLabels.app }}/{{ .GroupLabels.alertname }}
# 2.条件判断(运算符:gt/lt/eq/gte/lte)
  {{- if gt (len .Alerts.Firing) 0 -}}
# 3.条件循环
    {{- range $index, $alert := .Alerts -}}
        告警序号: {{ $index }}
        告警类型: {{ $alert.Labels.alertname }}
        告警级别: {{ $alert.Labels.severity }}
        告警详情: {{ $alert.Annotations.summary }}
# 4.此处格式的时间字符串是Go诞生时间。
        故障时间: {{ $alert.StartsAt.Format "2006-01-02 15:04:05" }}
        恢复时间: {{ $alert.EndsAt.Format "2006-01-02 15:04:05" }}
    {{ end }}
  {{ end }}
{{ end}}

# 实例instance标签
{{ .GroupLabels.instance }} 

# 告警数量
{{ .Alerts | len }} 
{{ (len .Alerts.Firing) }}

{{- range $index, $alert := .Alerts -}}
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}
告警详情: {{ $alert.Annotations.summary }}
故障时间: {{ $alert.StartsAt.Format "2006-01-02 15:04:05" }}
恢复时间: {{ $alert.EndsAt.Format "2006-01-02 15:04:05" }}
{{- end }}

# - 注意此处非全局所以没有smtp前缀
# Whether or not to notify about resolved alerts.
[ send_resolved: <boolean> | default = false ]

# The email address to send notifications to.
to: <tmpl_string>

# The sender address.
[ from: <tmpl_string> | default = global.smtp_from ]

# The SMTP host through which emails are sent.
[ smarthost: <string> | default = global.smtp_smarthost ]

# The hostname to identify to the SMTP server.
[ hello: <string> | default = global.smtp_hello ]

# SMTP authentication information.
[ auth_username: <string> | default = global.smtp_auth_username ]
[ auth_password: <secret> | default = global.smtp_auth_password ]
[ auth_secret: <secret> | default = global.smtp_auth_secret ]
[ auth_identity: <string> | default = global.smtp_auth_identity ]

# The SMTP TLS requirement.
# Note that Go does not support unencrypted connections to remote SMTP endpoints.
[ require_tls: <bool> | default = global.smtp_require_tls ]

# TLS configuration.
tls_config:
  [ <tls_config> ]

# The HTML body of the email notification.
[ html: <tmpl_string> | default = '{{ template "email.default.html" . }}' ]
# The text body of the email notification.
[ text: <tmpl_string> ]

# Further headers email header key/value pairs. Overrides any headers
# previously set by the notification implementation.
[ headers: { <string>: <tmpl_string>, ... } ]

global:
  scrape_interval: 2m
  scrape_timeout: 10s
  # - 每分钟进行报警规则的评估 (建议根据实际情况进行修改)
  evaluation_interval: 1m
  external_labels:
    monitor: 'prom-demo'

alerting:
  # - 设置报警管理系统的目标地址 (可以配置多个AlertManager)
  alertmanagers:
  - scheme: http
    static_configs:
    - targets:
      - '192.168.12.107:30093'

rule_files:
  # - 指标监控告警规则 (可以根据不同的场景进行报警规则)
  - /etc/prometheus/conf.d/rules/*.rules

  # - 采集监控的静态目标和自动化发现目标
scrape_configs:
  - job_name: 'prom-Server'
    static_configs:
      - targets: ['localhost:9090']
  - job_name: 'cAdvisor'
    static_configs:
      - targets: ['192.168.12.111:9100']
  - job_name: 'linux_exporter'
    scrape_interval: 30s
    file_sd_configs:
    - files:
      - /etc/prometheus/conf.d/discovery/k8s_nodes.yaml
      # - 自动刷新自动发现配置文件
      refresh_interval: 1m

tee alert.rules <<'EOF'
groups:
- name: node-normal
  rules:
  - alert: service_down
    expr: up {job="linux_exporter"} == 0
    # - 对于持续时间的设置在实际环境中至少建议5min，以减少噪声从而减轻固有监控各种情况,此处设置为30s便于报警情况的观察。
    for: 30s
    labels:
      severity: 'critical'
      team: node
    annotations:
      summary: "主机 {{ $labels.instance }} 监控服务已停止运行超过 15s！"
      description: "请系统管理员尽快进行人工干预处理！"
  - alert: high_load
    expr: node_load1 > 0.7
    for: 5m
    labels:
      severity: 'warning'
      team: node
    annotations:
      summary: "主机 {{ $labels.instance }} 高负载大于0.7以上运行超过 5m！"
EOF

tee alertmanager.yaml <<'EOF'
# - 全局配置
global:
  resolve_timeout: 5m
  smtp_from: 'monitor@weiyigeek.top'
  # - 企业邮箱 SMTP 服务地址
  smtp_smarthost: 'smtp.exmail.qq.com:465'
  smtp_auth_username: 'monitor@weiyigeek.top'
  # - 第三方登录企业邮箱的授权码
  smtp_auth_password: xxxxxxxxxxx'
  smtp_require_tls: false
  # smtp_hello: 'qq.com'
# - 设置报警的分发策略
route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 1m
  # - 如果在10分钟内还未解除报警则进行发送告警
  repeat_interval: 10m
  receiver: 'default-email'
# -  配置告警消息接受者信息
receivers:
- name: 'default-email'
  email_configs:
  - to: 'master@weiyigeek.top'
    send_resolved: true
# - 抑制规则配置
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'instance']
EOF

# - 重启容器
docker restart prometheus_alertmanager prometheus_server

# - 关闭192.168.12.109机器的node_exporter服务
ssh -p 20211 weiyigeek@weiyigeek-109 "sudo -S systemctl stop node_exporter.service"
[sudo] password for weiyigeek:

linux_exporter (5/6 up)
Endpoint	State	 Error
http://192.168.12.109:9100/metrics	DOWN	Get "http://192.168.12.109:9100/metrics": dial tcp 192.168.12.109:9100: connect: connection refused

ssh -p 20211 weiyigeek@weiyigeek-109 "sudo -S systemctl start node_exporter.service"

tee email.tmpl<<'EOF'
{{ define "email.from" }}monitor@weiyigeek.top{{ end }}
{{ define "email.to" }}master@weiyigeek.top{{ end }}
{{ define "email.to.html" }}
{{ range .Alerts }}
<b>=========start==========</b><br>
告警程序: prometheus_alert <br>
告警级别: {{ .Labels.severity }} 级 <br>
告警类型: {{ .Labels.alertname }} <br>
故障主机: {{ .Labels.instance }} <br>
告警主题: {{ .Annotations.summary }} <br>
告警详情: {{ .Annotations.description }} <br>
触发时间: {{ .StartsAt.Format "2006-01-02 15:04:05" }} <br>
<b>=========end==========</b><br>
{{ end }}
{{ end }}
EOF

tee alertmanager.yaml <<'EOF'
global:
  resolve_timeout: 5m
  smtp_from: 'monitor@weiyigeek.top'
  smtp_smarthost: 'smtp.exmail.qq.com:465'
  smtp_auth_username: 'monitor@weiyigeek.top'
  smtp_auth_password: 'xxxxxxxxxxxxxxx'
  smtp_require_tls: false
  # smtp_hello: 'qq.com'

# - Tips 上面的全局邮件参数无法从email.tmpl中读取{{ template "email.username" . }}坑(排错一下午)
templates:
- '/alertmanager/email.tmpl'

route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 1m
  repeat_interval: 10m
  receiver: 'default-email'

receivers:
- name: 'default-email'
  email_configs:
  - to: '{{ template "email.to" . }}'
    html: '{{ template "email.to.html" . }}'
    # 如果问题已解决将发送信息如设置false则不发送
    send_resolved: true

inhibit_rules:
- source_match:
    severity: 'critical'
  target_match:
    severity: 'warning'
  equal: ['alertname', 'instance']
EOF

温馨提示：全局中的邮件参数无法从email.tmpl中读取定义的模板参数，例如{{ template "email.username" .}}坑(排错一下午)只有以下对象可以读取:

email_config:
- to: <tmpl_string>
  from: <tmpl_string>
  html: <tmpl_string>  | text: <tmpl_string>  {{ template "email.default.html" . }}

# Whether or not to notify about resolved alerts.
[ send_resolved: <boolean> | default = false ]

# The WeChat API URL.
[ api_url: <string> | default = global.wechat_api_url ]

# 企业微信账号唯一 ID， 可以在我的企业中查看。
[ corp_id: <string> | default = global.wechat_api_corp_id ]
# 第三方企业应用的密钥，可以在自己创建的第三方企业应用详情页面查看。
[ api_secret: <secret> | default = global.wechat_api_secret ]
# 第三方企业应用的 ID，可以在自己创建的第三方企业应用详情页面查看。
[ agent_id: <string> | default = '{{ template "wechat.default.agent_id" . }}' ]
# 在后台通讯录查看需要发送的组的部门ID ( PartyID1 | PartyID2)
[ to_party: <string> | default = '{{ template "wechat.default.to_party" . }}' ]
# 发送的成员 @all 表示所有的成员
[ to_user: <string> | default = '{{ template "wechat.default.to_user" . }}' ]
# 发送给特点 tag 标记的，
[ to_tag: <string> | default = '{{ template "wechat.default.to_tag" . }}' ]
# 
[ message: <tmpl_string> | default = '{{ template "wechat.default.message" . }}' ]

# - 例如:博主的AgentId和Secret的格式如下
企业ID: ww32cf0bf4e4b11487
agent_id: 1000002
api_secret: SYCdXbWj5SjdT9pOSQnjIZ4RF_SIRqERVuZNWE1yE5g
to_party: 1

tee alertmanager.yaml <<'EOF'
global:
  resolve_timeout: 2m

templates:
- '/alertmanager/*.tmpl'

route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 1m
  repeat_interval: 10m
  receiver: 'wechat'

receivers:
- name: 'default-email'
  email_configs:
  - from: 'monitor@weiyigeek.top'
    smarthost: 'smtp.exmail.qq.com:465'
    auth_username: 'monitor@weiyigeek.top'
    auth_password: 'xxxxxxxxxxxxxxx'
    require_tls: false
    # smtp_hello: 'qq.com'
    to: '{{ template "email.to" . }}'
    html: '{{ template "email.to.html" . }}'
    send_resolved: true
  
- name: 'wechat'
  wechat_configs:
  - send_resolved: true
    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'
    corp_id: 'ww32cf0bf4e4b11487'
    agent_id: '1000002'
    api_secret: 'SYCdXbWj5SjdT9pOSQnjIZ4RF_SIRqERVuZNWE1yE5g'
    to_user: '@all'
    to_party: '1'
    message: '{{ template "wechat.default.message" . }}'
   
inhibit_rules:
- source_match:
    severity: 'critical'
  target_match:
    severity: 'warning'
  equal: ['alertname', 'instance']
EOF

tee wechat.tmpl <<'EOF'
{{ define "wechat.default.message" }}
{{- if gt (len .Alerts.Firing) 0 -}}
{{- range $index, $alert := .Alerts -}}
{{- if eq $index 0 -}}
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}
=====================
{{- end }}
===告警详情===
告警详情: {{ $alert.Annotations.summary }}
告警描述: {{ $alert.Annotations.description }}
故障时间: {{ $alert.StartsAt.Format "2006-01-02 15:04:05" }}
===参考信息===
{{ if gt (len $alert.Labels.instance) 0 -}}故障实例ip: {{ $alert.Labels.instance }};{{- end -}}
{{- if gt (len $alert.Labels.namespace) 0 -}}故障实例所在namespace: {{ $alert.Labels.namespace }};{{- end -}}
{{- if gt (len $alert.Labels.node) 0 -}}故障物理机ip: {{ $alert.Labels.node }};{{- end -}}
{{- if gt (len $alert.Labels.pod_name) 0 -}}故障pod名称: {{ $alert.Labels.pod_name }}{{- end }}
=====================
{{- end }}
{{- end }}

{{- if gt (len .Alerts.Resolved) 0 -}}
{{- range $index, $alert := .Alerts -}}
{{- if eq $index 0 -}}
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}
=====================
{{- end }}
===告警详情===
告警详情: {{ $alert.Annotations.summary }}
故障时间: {{ $alert.StartsAt.Format "2006-01-02 15:04:05" }}
恢复时间: {{ $alert.EndsAt.Format "2006-01-02 15:04:05" }}
===参考信息===
{{ if gt (len $alert.Labels.instance) 0 -}}故障实例ip: {{ $alert.Labels.instance }};{{- end -}}
{{- if gt (len $alert.Labels.namespace) 0 -}}故障实例所在namespace: {{ $alert.Labels.namespace }};{{- end -}}
{{- if gt (len $alert.Labels.node) 0 -}}故障物理机ip: {{ $alert.Labels.node }};{{- end -}}
{{- if gt (len $alert.Labels.pod_name) 0 -}}故障pod名称: {{ $alert.Labels.pod_name }};{{- end }}
=====================
{{- end }}
{{- end }}
{{- end }}
EOF

# message: '{{ template "wechat.default.message" . }}'
apt 
# 重启 
docker restart prometheus_alertmanager
# 日志查看
docker logs --tail 50 -f prometheus_alertmanager

{{ define "wechat.default.message" }}
{{- if gt (len .Alerts.Firing) 0 -}}
{{- range $index, $alert := .Alerts -}}
{{- if eq $index 0 }}
==========异常告警==========
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}
告警详情: {{ $alert.Annotations.message }}{{ $alert.Annotations.description}};{{$alert.Annotations.summary}}
故障时间: {{ ($alert.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}
{{- if gt (len $alert.Labels.instance) 0 }}
实例信息: {{ $alert.Labels.instance }}
{{- end }}
{{- if gt (len $alert.Labels.namespace) 0 }}
命名空间: {{ $alert.Labels.namespace }}
{{- end }}
{{- if gt (len $alert.Labels.node) 0 }}
节点信息: {{ $alert.Labels.node }}
{{- end }}
{{- if gt (len $alert.Labels.pod) 0 }}
实例名称: {{ $alert.Labels.pod }}
{{- end }}
============END============
{{- end }}
{{- end }}
{{- end }}
{{- if gt (len .Alerts.Resolved) 0 -}}
{{- range $index, $alert := .Alerts -}}
{{- if eq $index 0 }}
==========异常恢复==========
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}
告警详情: {{ $alert.Annotations.message }}{{ $alert.Annotations.description}};{{$alert.Annotations.summary}}
故障时间: {{ ($alert.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}
恢复时间: {{ ($alert.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}
{{- if gt (len $alert.Labels.instance) 0 }}
实例信息: {{ $alert.Labels.instance }}
{{- end }}
{{- if gt (len $alert.Labels.namespace) 0 }}
命名空间: {{ $alert.Labels.namespace }}
{{- end }}
{{- if gt (len $alert.Labels.node) 0 }}
节点信息: {{ $alert.Labels.node }}
{{- end }}
{{- if gt (len $alert.Labels.pod) 0 }}
实例名称: {{ $alert.Labels.pod }}
{{- end }}
============END============
{{- end }}
{{- end }}
{{- end }}
{{- end }}

# 镜像拉取与镜像运行
docker pull nginx:1.21.0
docker run --name auth -d nginx:1.21.0
docker exec -it auth bash

# 使用 `apache2-utils` 提供的 `htpasswd` 工具创建一个用户文件,该镜像中没自带所以我们下载即可
apt install apache2-utils
# 运行 htpasswd 生成一个或多个认证用户
htpasswd -bc htpasswd.auth weiyigeek password
Adding password for user weiyigeek
htpasswd -b htpasswd.auth prometheus password
Adding password for user prometheus
htpasswd -b htpasswd.auth monitor password
Adding password for user monitor

# 拷贝容器中nginx配置
docker cp auth:/etc/nginx /monitor/nginx

# 拷贝生成htpasswd.auth 到宿主机
docker cp auth:/tmp/htpasswd.auth /monitor/nginx

# Nginx 配置文件
root@prometheus:/monitor/nginx/conf# cat conf.d/default.conf
server {
  listen       8080;
  listen  [::]:8080;
  server_name  monitor.weiyigeek.top;

  #access_log  /var/log/nginx/host.access.log  main;
  add_header X-Frame-Options SAMEORIGIN;
  auth_basic "weiyigeek Prometheus Metrics";
  auth_basic_user_file /etc/nginx/auth/htpasswd.auth;
  #这里放的是认证文件位置

  location / {
    proxy_pass http://monitor.weiyigeek.top:9090;
    proxy_redirect          off;
    proxy_set_header        X-Real-IP       $remote_addr;
    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header        Host            $host;
    proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504 http_404;
    access_log  /var/log/nginx/prometheus_access.log;
  }
}

# - 容器启动
docker run -d --name=prometheus_proxy -p 8080:8080 -v /monitor/nginx/conf:/etc/nginx -v /monitor/nginx/logs:/var/log/nginx/ nginx:1.21.0

# - 容器状态
docker ps
  <!-- CONTAINER ID   IMAGE                   COMMAND                  CREATED         STATUS         PORTS                                               NAMES
  6347a6498e69   nginx:1.21.0            "/docker-entrypoint.…"   6 minutes ago   Up 6 minutes   80/tcp, 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp   prometheus_proxy -->

# 启动防火墙 & 查看状态
systemctl enable ufw && systemctl restart ufw
ufw status && systemctl status ufw

# 允许所有的机器访问 80(Grafana),8080(Nginx Proxy Prometheus Server)
sudo ufw allow proto tcp to port 80,8080

# 可以通过ip addr查看网卡的网段以便后续docker或者本地访问。
ip addr

# 只允许从本地和docker0网卡访问 9090,9093,9094,9115 端口
sudo ufw allow proto tcp from 127.0.0.1 to any port 9090,9093,9094,9115
ufw delete allow proto tcp from 172.17.0.1/16 to any port 9090,9093,9094,9115

usage: promtool [<flags>] <command> [<args> ...]

Flags:
-h, --help     Show context-sensitive help (also try --help-long and --help-man).

Commands:
  help [<command>...] Show help.

  check config <config-files>... # Check if the config files are valid or not.
  check web-config <web-config-files>... # Check if the web config files are valid or not.
  check rules <rule-files>... # Check if the rule files are valid or not.
  check metrics # Pass Prometheus metrics over stdin to lint them for consistency and correctness.

  query instant [<flags>] <server> <expr>  # Run instant query.
  query range [<flags>] <server> <expr> # Run range query.
  query series --match=MATCH [<flags>] <server> #  Run series query.
  query labels [<flags>] <server> <name> # Run labels query.

  debug pprof <server> # Fetch profiling debug information.
  debug metrics <server> # Fetch metrics debug information.
  debug all <server> # Fetch all debug information.

  test rules <test-rule-file>... # Unit tests for rules.

  tsdb bench write [<flags>] [<file>] # Run a write performance benchmark.
  tsdb analyze [<flags>] [<db path>] [<block id>] #  Analyze churn, label pair cardinality.
  tsdb list [<flags>] [<db path>] #  List tsdb blocks.
  tsdb dump [<flags>] [<db path>] # Dump samples from a TSDB.
  tsdb create-blocks-from openmetrics <input file> [<output directory>] #  Import samples from OpenMetrics input and produce TSDB blocks. Please refer to the storage docs for more details.
  tsdb create-blocks-from rules --start=START [<flags>] <rule-files>... # Create blocks of data for new recording rules.

# - 1.验证指标输出是否有效并执行格式检查。
$ cat metrics.prom | promtool check metrics
$ curl -s http://localhost:9090/metrics | promtool check metrics