监控阈值设置问题：监控阈值设置不当，导致误报或漏报

# 示例：CPU 使用率超过 80% 触发告警 
groups:
- name: example
  rules:
  - alert: HighCpuUsage
    expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High CPU usage on {{ $labels.instance  }}"
      description: "CPU usage is above 80% for more than 5 minutes."

# 示例：磁盘空间不足触发告警 
Trigger: {Template OS Linux:vfs.fs.size[/,pfree].last()}<20 
Severity: Warning

# 示例：预测未来 1 小时的磁盘使用情况
groups:
- name: example
  rules:
  - alert: DiskSpacePredictedLow
    expr: predict_linear(node_filesystem_free_bytes{mountpoint="/"}[1h], 3600) < 0
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "Disk space predicted to be low on {{ $labels.instance  }}"
      description: "Disk space is predicted to be exhausted within the next hour."

# 示例：计算过去 7 天的平均磁盘使用率
sar -d -f /var/log/sa/sa08 | awk '/Average:/ {sum += $5; count++} END {print sum/count}'

# 示例：查询过去 7 天的 CPU 使用率分布
{
  "query": {
    "range": {
      "@timestamp": {
        "gte": "now-7d",
        "lte": "now"
      }
    }
  },
  "aggs": {
    "cpu_usage_stats": {
      "stats": {
        "field": "cpu_usage"
      }
    }
  }
}

#!/usr/bin/env python3 
 
import pandas as pd
 
# 加载历史数据
data = pd.read_csv('/path/to/cpu_usage.csv') 
 
# 计算统计信息
mean = data['cpu_usage'].mean()
std_dev = data['cpu_usage'].std()
 
# 设置阈值
threshold_high = mean + 2 * std_dev
threshold_low = mean - 2 * std_dev 
 
print(f"建议的高阈值：{threshold_high}")
print(f"建议的低阈值：{threshold_low}")

# 模拟高负载 
stress-ng --cpu 4 --timeout 60s
 
# 模拟磁盘空间不足
dd if=/dev/zero of=/mnt/testfile bs=1G count=10

#!/bin/bash
 
# 测试阈值设置
test_threshold() {
    echo "开始测试阈值设置..."
    stress-ng --cpu 4 --timeout 60s
    if [ $? -eq 0 ]; then 
        echo "测试成功" >> /var/log/threshold_test.log 
    else
        echo "测试失败" >> /var/log/threshold_test.log 
    fi 
}
 
test_threshold