shell 脚本监控k8s预警

0 */1 * * * cd  /srv/monitor/k8s_monitor_pod.sh 

#!/bin/bash

# 定义阈值
CPU_THRESHOLD=85
MEMORY_THRESHOLD=95

# 定义命名空间列表
NAMESPACES=("bos" "gts" "wallet")

# 循环遍历命名空间
for NAMESPACE in "${NAMESPACES[@]}"; do
    # 获取命名空间中的所有Pod名称
    POD_NAMES=$(/usr/local/bin/kubectl get pod -n $NAMESPACE | awk '{print $1}' | tail -n +2)

    # 循环遍历Pod名称
    for POD_NAME in $POD_NAMES; do
        # 获取Pod的CPU和内存使用情况
        CPU_STATS=$(/usr/local/bin/kubectl top pod -n $NAMESPACE | grep $POD_NAME | awk '{print $2}' | sed 's/m$//')
        MEM_STATS=$(/usr/local/bin/kubectl top pod -n $NAMESPACE | grep $POD_NAME | awk '{print $3}' | sed 's/Mi$//')

        # 获取Pod的CPU和内存限制
        CPU_LIMITS=$(/usr/local/bin/kubectl describe pod $POD_NAME -n $NAMESPACE | grep cpu | head -n 1 | awk '{print $2}')
        MEM_LIMITS=$(/usr/local/bin/kubectl describe pod $POD_NAME -n $NAMESPACE | grep memory | head -n 1 | awk '{print $2}')

        # 处理CPU限制
        if [[ "$CPU_LIMITS" != *"m"* ]]; then
            CPU_LIMITS_END=$(echo "$CPU_LIMITS * 1024" | bc)
        else
            CPU_LIMITS_END=$(echo "$CPU_LIMITS" | sed 's/m$//')
        fi

        # 处理内存限制
        if [[ "$MEM_LIMITS" != *"Mi"* ]]; then
            MEMORY_LIMITS_SECOND=$(echo "$MEM_LIMITS" | sed 's/Gi$//')
            MEMORY_LIMITS_END=$(echo "$MEMORY_LIMITS_SECOND * 1024" | bc)
        else
            MEMORY_LIMITS_END=$(echo "$MEM_LIMITS" | sed 's/Mi$//')
        fi

        # 计算CPU和内存使用率
        CPU_PERCENTAGE=$(echo "scale=2; $CPU_STATS / $CPU_LIMITS_END * 100" | bc -l)
        MEMORY_PERCENTAGE=$(echo "scale=2; $MEM_STATS / $MEMORY_LIMITS_END * 100" | bc -l)

        # 检查CPU使用率是否超过阈值
        if (($(echo "$CPU_PERCENTAGE > $CPU_THRESHOLD" | bc -l))); then
            echo -e "\e[31mPOD_NAME: $POD_NAME CPU_LIMITS: $CPU_LIMITS $CPU_PERCENTAGE MEM_LIMITS: $MEM_LIMITS $MEMORY_PERCENTAGE\e[0m"
            # 发送告警通知到Lark Webhook机器人
            curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"text","content":{"text":"告警：'"$POD_NAME"'资源使用率过高，CPU使用率：'"$CPU_PERCENTAGE"'%"}}' https://open.larksuite.com/open-apis/bot/v2/hook/xxxxxxxx
        else
            # 打印CPU_LIMITS和MEM_LIMITS的值
            echo POD_NAME: $POD_NAME "CPU_LIMITS: $CPU_LIMITS" $CPU_PERCENTAGE "MEM_LIMITS: $MEM_LIMITS" $MEMORY_PERCENTAGE
        fi

        # 检查内存使用率是否超过阈值
        if (($(echo "$MEMORY_PERCENTAGE > $MEMORY_THRESHOLD" | bc -l))); then
            echo -e "\e[31mPOD_NAME: $POD_NAME CPU_LIMITS: $CPU_LIMITS $CPU_PERCENTAGE MEM_LIMITS: $MEM_LIMITS $MEMORY_PERCENTAGE\e[0m"
            # 发送告警通知到Lark Webhook机器人
            curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"text","content":{"text":"告警：'"$POD_NAME"'资源使用率过高，内存使用率：'"$MEMORY_PERCENTAGE"'%"}}' https://open.larksuite.com/open-apis/bot/v2/hook/xxxxxxxx
        else
            # 打印CPU_LIMITS和MEM_LIMITS的值
            echo POD_NAME: $POD_NAME "CPU_LIMITS: $CPU_LIMITS" $CPU_PERCENTAGE "MEM_LIMITS: $MEM_LIMITS_FIRST" $MEMORY_PERCENTAGE
        fi
    done
done