136_生产监控：Prometheus集成 - 设置警报与指标选择与LLM部署监控最佳实践

监控对象 → 数据采集层 → 存储层 → 分析查询层 → 可视化/告警层

# 全局配置
global:
  scrape_interval: 15s      # 抓取间隔
  evaluation_interval: 15s  # 规则评估间隔

# 告警管理配置
alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

# 告警规则文件
rule_files:
  - "alert_rules.yml"

# 抓取配置
scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node_exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'gpu_exporter'
    static_configs:
      - targets: ['gpu-exporter:9400']

  - job_name: 'llm_service'
    metrics_path: '/metrics'
    scrape_interval: 5s      # LLM服务指标抓取更频繁
    static_configs:
      - targets: ['llm-service:8000']

# prometheus.yml中的Node Exporter配置
srape_configs:
  - job_name: 'node_exporter'
    scrape_interval: 10s
    static_configs:
      - targets: ['node-exporter:9100']
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        regex: '(.*):9100'
        replacement: '${1}'

# CPU使用率
node_cpu_seconds_total{mode!="idle"}

# 内存使用率
node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes

# 磁盘使用率
node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}

# 网络流量
rate(node_network_receive_bytes_total{device!="lo"}[1m])
rate(node_network_transmit_bytes_total{device!="lo"}[1m])

# prometheus.yml中的GPU Exporter配置
srape_configs:
  - job_name: 'gpu_exporter'
    scrape_interval: 5s  # GPU指标变化快，抓取更频繁
    static_configs:
      - targets: ['gpu-exporter:9400']
    metrics_relabel_configs:
      - source_labels: [__name__]
        regex: 'nvidia_smi_.*'
        action: keep

# GPU利用率
nvidia_smi_gpu_utilization

# GPU内存使用率
nvidia_smi_gpu_memory_used_bytes / nvidia_smi_gpu_memory_total_bytes

# GPU温度
nvidia_smi_gpu_temperature

# GPU功耗
nvidia_smi_gpu_power_usage

# requirements.txt
fastapi==0.104.1
uvicorn==0.24.0.post1
prometheus-client==0.17.1

# llm_service.py
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
import time
from prometheus_client import Counter, Histogram, Gauge, start_http_server
import threading

# 创建FastAPI应用
app = FastAPI()

# 定义Prometheus指标
REQUEST_COUNT = Counter(
    'llm_requests_total', 
    'Total number of LLM requests',
    ['endpoint', 'model', 'status']
)

REQUEST_LATENCY = Histogram(
    'llm_request_latency_seconds',
    'LLM request latency in seconds',
    ['endpoint', 'model'],
    buckets=(0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0)
)

ACTIVE_REQUESTS = Gauge(
    'llm_active_requests',
    'Number of active LLM requests',
    ['endpoint', 'model']
)

TOKENS_PROCESSED = Counter(
    'llm_tokens_processed_total',
    'Total number of tokens processed',
    ['type', 'model']  # type: input/output
)

GPU_MEMORY_USAGE = Gauge(
    'llm_gpu_memory_usage_bytes',
    'GPU memory usage for LLM inference',
    ['gpu_id']
)

# 中间件：记录请求指标
@app.middleware("http")
async def prometheus_middleware(request: Request, call_next):
    endpoint = request.url.path
    model = request.query_params.get("model", "default")
    
    # 增加活跃请求数
    ACTIVE_REQUESTS.labels(endpoint=endpoint, model=model).inc()
    
    # 记录请求开始时间
    start_time = time.time()
    
    try:
        # 处理请求
        response = await call_next(request)
        status = str(response.status_code)
        
        # 记录请求计数
        REQUEST_COUNT.labels(endpoint=endpoint, model=model, status=status).inc()
        
        return response
    except Exception as e:
        # 记录错误
        REQUEST_COUNT.labels(endpoint=endpoint, model=model, status="error").inc()
        return JSONResponse(
            status_code=500,
            content={"detail": str(e)}
        )
    finally:
        # 计算延迟并记录
        latency = time.time() - start_time
        REQUEST_LATENCY.labels(endpoint=endpoint, model=model).observe(latency)
        
        # 减少活跃请求数
        ACTIVE_REQUESTS.labels(endpoint=endpoint, model=model).dec()

# LLM推理端点
@app.post("/api/v1/generate")
async def generate_text(request: dict):
    model = request.get("model", "default")
    prompt = request.get("prompt", "")
    max_tokens = request.get("max_tokens", 100)
    
    # 记录输入token数（简化计算）
    input_tokens = len(prompt.split())
    TOKENS_PROCESSED.labels(type="input", model=model).inc(input_tokens)
    
    # 模拟LLM推理过程
    # 实际应用中这里会调用真实的模型推理
    time.sleep(0.5)  # 模拟延迟
    
    # 模拟输出
    output = f"Response to your prompt: {prompt[:20]}..."
    output_tokens = len(output.split())
    
    # 记录输出token数
    TOKENS_PROCESSED.labels(type="output", model=model).inc(output_tokens)
    
    # 模拟GPU内存使用（实际应用中应从GPU监控库获取）
    GPU_MEMORY_USAGE.labels(gpu_id="0").set(1024 * 1024 * 1024 * 12)  # 12GB
    
    return {
        "model": model,
        "output": output,
        "tokens": {
            "input": input_tokens,
            "output": output_tokens
        }
    }

# 健康检查端点
@app.get("/health")
async def health_check():
    return {"status": "healthy"}

# 在单独线程中启动Prometheus指标服务器
def start_metrics_server():
    start_http_server(8000)

# 启动指标服务器
threading.Thread(target=start_metrics_server, daemon=True).start()

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8080)

# service-monitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: llm-service-monitor
  namespace: monitoring
  labels:
    release: prometheus

spec:
  selector:
    matchLabels:
      app: llm-service
  namespaceSelector:
    matchNames:
      - default
  endpoints:
    - port: metrics
      interval: 5s
      path: /metrics
      scrapeTimeout: 4s

# llm-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: llm-service
  labels:
    app: llm-service
spec:
  selector:
    app: llm-service
  ports:
    - name: http
      port: 8080
    - name: metrics
      port: 8000