监控报警问题：监控报警设置不当，错过重要警报

# 示例：CPU 使用率超过 80% 触发告警
groups:
- name: example 
  rules:
  - alert: HighCpuUsage 
    expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High CPU usage on {{ $labels.instance  }}"
      description: "CPU usage is above 80% for more than 5 minutes."

# 示例：磁盘空间不足触发告警 
Trigger: {Template OS Linux:vfs.fs.size[/,pfree].last()}<10 
Severity: High

# 示例：计算过去 7 天的平均磁盘使用率
sar -d -f /var/log/sa/sa08 | awk '/Average:/ {sum += $5; count++} END {print sum/count}'

# 示例：Prometheus Alertmanager 配置
route:
  receiver: 'email-notifications'
 
receivers:
- name: 'email-notifications'
  email_configs:
  - to: 'admin@example.com' 
    from: 'alertmanager@example.com' 
    smarthost: 'smtp.example.com:587' 
    auth_username: 'alertmanager@example.com' 
    auth_password: 'password'

# 示例：Prometheus Alertmanager 配置 
receivers:
- name: 'slack-notifications'
  slack_configs:
  - api_url: 'https://hooks.slack.com/services/your/webhook/url' 
    channel: '#alerts'
    send_resolved: true

# 示例：Twilio 发送电话报警 
curl -X POST https://api.twilio.com/2010-04-01/Accounts/{AccountSID}/Calls.json  \
--data-urlencode "To=+1234567890" \
--data-urlencode "From=+0987654321" \
--data-urlencode "Url=http://example.com/alert_message.xml"  \
-u {AccountSID}:{AuthToken}

# 示例：仅报警生产环境的服务器 
up{environment="production"}

# 示例：使用 ELK Stack 分析报警日志
filter {
    if [message] =~ /ERROR/ {
        mutate {
            add_field => { "severity" => "high" }
        }
    }
}

# 模拟高负载 
stress-ng --cpu 4 --timeout 60s
 
# 模拟网络中断 
iptables -A INPUT -p tcp --dport 80 -j DROP

#!/bin/bash
 
# 测试报警设置 
test_alerting() {
    echo "开始测试报警设置..."
    stress-ng --cpu 4 --timeout 60s
    if [ $? -eq 0 ]; then
        echo "测试成功" >> /var/log/alerting_test.log 
    else 
        echo "测试失败" >> /var/log/alerting_test.log 
    fi
}
 
test_alerting