构建AI智能体：模型评估指南：准确率、精确率、F1分数与ROC/AUC的深度解析

import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from sklearn.metrics import confusion_matrix
plt.rcParams['font.sans-serif'] = ['Microsoft YaHei', 'SimHei', 'Segoe UI Symbol', 'DejaVu Sans']
plt.rcParams['axes.unicode_minus'] = False
# 数据
y_true = [0, 0, 1, 1, 0, 1, 0, 1, 1, 0]
y_pred = [0, 1, 1, 0, 0, 1, 0, 1, 0, 0]

# 计算混淆矩阵
cm = confusion_matrix(y_true, y_pred)

# 简单绘图
fig, ax = plt.subplots(figsize=(6, 5))

# 为每个格子自定义颜色（示例：二分类的TN/FP/FN/TP）
cell_colors = {
    (0, 0): 'lightblue',   # TN
    (0, 1): 'lightcoral',  # FP
    (1, 0): 'gold',        # FN
    (1, 1): 'lightgreen'   # TP
}

# 构建颜色索引矩阵
unique_colors = list(dict.fromkeys(cell_colors.values()))
color_to_idx = {c: i for i, c in enumerate(unique_colors)}
color_idx = np.zeros_like(cm, dtype=int)
for (r, c), col in cell_colors.items():
    color_idx[r, c] = color_to_idx[col]

from matplotlib.colors import ListedColormap
cmap = ListedColormap(unique_colors)

# 使用颜色矩阵绘制背景，并在其上叠加真实的混淆矩阵数值
im = ax.imshow(color_idx, cmap=cmap, vmin=0, vmax=max(color_to_idx.values()))

# 在每个格子写入数值
for i in range(cm.shape[0]):
    for j in range(cm.shape[1]):
        ax.text(j, i, f"{cm[i, j]}", ha='center', va='center', color='black', fontsize=12)

ax.set_title('混淆矩阵')
ax.set_xlabel('预测值')
ax.set_ylabel('真实值')
ax.set_xticks(np.arange(cm.shape[1]))
ax.set_yticks(np.arange(cm.shape[0]))
plt.show()

# 打印结果
print("真实值:", y_true)
print("预测值:", y_pred)
print("混淆矩阵:")
print(cm)

# 手动计算过程
tp = 0  # 真正例
fp = 0  # 假正例  
fn = 0  # 假负例
tn = 0  # 真负例

print("样本逐个分析:")
for i, (true, pred) in enumerate(zip(y_true, y_pred)):
    if true == 1 and pred == 1:
        tp += 1
        result = "TP (真正例)"
    elif true == 0 and pred == 1:
        fp += 1
        result = "FP (假正例)"
    elif true == 1 and pred == 0:
        fn += 1
        result = "FN (假负例)"
    else:  # true == 0 and pred == 0
        tn += 1
        result = "TN (真负例)"
    
    print(f"样本{i+1}: {true} -> {pred} | {result}")

print(f"\n📊 最终统计:")
print(f"TP (真正例): {tp}")
print(f"FP (假正例): {fp}") 
print(f"FN (假负例): {fn}")
print(f"TN (真负例): {tn}")

# 创建对比表格
fig, ax = plt.subplots(figsize=(12, 4))
ax.axis('tight')
ax.axis('off')

# 表格数据
table_data = []
headers = ['样本', '真实值', '预测值', '比较结果', '类别']
table_data.append(headers)

for i, (true, pred) in enumerate(zip(y_true, y_pred)):
    if true == pred:
        if true == 1:
            category = "TP (真正例)"
            color = "green"
        else:
            category = "TN (真负例)" 
            color = "blue"
    else:
        if pred == 1:
            category = "FP (假正例)"
            color = "orange"
        else:
            category = "FN (假负例)"
            color = "red"
    
    comparison = "正确" if true == pred else "错误"
    table_data.append([f"样本{i+1}", true, pred, comparison, category])

# 创建表格
table = ax.table(cellText=table_data, 
                cellLoc='center', 
                loc='center',
                colWidths=[0.15, 0.15, 0.15, 0.2, 0.35])

# 设置表格样式
table.auto_set_font_size(False)
table.set_fontsize(11)
table.scale(1, 1.6)

# 为不同类别设置颜色
for i in range(1, len(table_data)):
    # 类别列着色
    cell = table[i, 4]  # 类别列
    if "TP" in table_data[i][4]:
        cell.set_facecolor('lightgreen')
    elif "TN" in table_data[i][4]:
        cell.set_facecolor('lightblue') 
    elif "FP" in table_data[i][4]:
        cell.set_facecolor('lightcoral')
    elif "FN" in table_data[i][4]:
        cell.set_facecolor('gold')

    # 比较结果列着色（正确为绿色，错误为红色）
    comp_cell = table[i, 3]  # 比较结果列
    if table_data[i][3] == "正确":
        comp_cell.set_facecolor('palegreen')
        comp_cell.get_text().set_color('green')
    else:  # 错误
        comp_cell.set_facecolor('mistyrose')
        comp_cell.get_text().set_color('red')

plt.title('混淆矩阵计算过程 - 逐样本分析', fontsize=14, pad=2)
plt.tight_layout(pad=1.5)
plt.show()

accuracy_manual = (tp + tn) / (tp + tn + fp + fn)
print(f"\n1. 准确率 (Accuracy):")
print(f"   公式: (TP + TN) / (TP + TN + FP + FN)")
print(f"   计算: ({tp} + {tn}) / {len(y_true)} = {accuracy_manual:.3f}")
print(f"   含义: 模型整体预测正确率")

precision_manual = tp / (tp + fp) if (tp + fp) > 0 else 0
print(f"\n精确率 (Precision):")
print(f"  公式: TP / (TP + FP)")
print(f"  计算: {tp} / ({tp} + {fp}) = {precision_manual:.4f}")

recall_manual = tp / (tp + fn) if (tp + fn) > 0 else 0
print(f"\n召回率 (Recall):")
print(f"  公式: TP / (TP + FN)")
print(f"  计算: {tp} / ({tp} + {fn}) = {recall_manual:.4f}")

f1_manual = 2 * (precision_manual * recall_manual) / (precision_manual + recall_manual) if (precision_manual + recall_manual) > 0 else 0
print(f"\nF1分数 (F1-Score):")
print(f"  公式: 2 * (Precision * Recall) / (Precision + Recall)")
print(f"  计算: 2 * ({precision_manual:.4f} * {recall_manual:.4f}) / ({precision_manual:.4f} + {recall_manual:.4f}) = {f1_manual:.4f}")

# 2. 指标对比条形图与指标关系图
fig_metrics, axes_metrics = plt.subplots(1, 2, figsize=(12, 5))

# 指标对比条形图
ax_bar = axes_metrics[0]
metrics = ['准确率', '精确率', '召回率', 'F1分数']
values = [accuracy_manual, precision_manual, recall_manual, f1_manual]
colors = ['#3498db', '#e74c3c', '#2ecc71', '#f39c12']

bars = ax_bar.bar(metrics, values, color=colors, alpha=0.7, width=0.6)
ax_bar.set_ylim(0, 1)
ax_bar.set_title('核心评估指标对比', fontsize=14, pad=20)
ax_bar.set_ylabel('分数', fontsize=12)
ax_bar.grid(True, alpha=0.3, axis='y')

# 在条形上添加数值
for bar, value in zip(bars, values):
    height = bar.get_height()
    ax_bar.text(bar.get_x() + bar.get_width()/2., height + 0.02,
                f'{value:.3f}', ha='center', va='bottom', fontsize=11, fontweight='bold')

# 指标关系图（混淆矩阵分布）
ax_pie = axes_metrics[1]
categories = ['TN', 'FP', 'FN', 'TP']
counts = [tn, fp, fn, tp]
colors_detail = ['lightblue', 'lightcoral', 'gold', 'lightgreen']
wedges, texts, autotexts = ax_pie.pie(counts, labels=categories, colors=colors_detail,
                                      autopct='%1.0f%%', startangle=90)
ax_pie.axis('equal')
ax_pie.set_title('混淆矩阵分布', fontsize=14, pad=20)
plt.tight_layout()
plt.show()

from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score, roc_curve, auc
# 计算ROC曲线和AUC
# 模拟预测概率（用于ROC曲线）
y_pred_proba = [0.1, 0.7, 0.8, 0.4, 0.2, 0.9, 0.3, 0.85, 0.6, 0.25]
fpr, tpr, thresholds = roc_curve(y_true, y_pred_proba)
roc_auc = auc(fpr, tpr)

print("\n" + "="*50)
print(" ROC曲线与AUC计算:")
print("="*50)

print(f"\nROC曲线坐标点 (FPR, TPR):")
for i, (f, t, thresh) in enumerate(zip(fpr, tpr, thresholds)):
    print(f"  阈值={thresh:.2f}: FPR={f:.2f}, TPR={t:.2f}")

print(f"\nAUC (曲线下面积) = {roc_auc:.3f}")
print("AUC含义:")
print("  • 0.9-1.0: 极好的分类能力")
print("  • 0.8-0.9: 良好的分类能力") 
print("  • 0.7-0.8: 一般的分类能力")
print("  • 0.5-0.7: 较差的分类能力")
print("  • 0.5: 随机猜测")

# 创建可视化图形
fig, axes = plt.subplots(1, 3, figsize=(15, 5))

# 4. ROC曲线
plt.sca(axes[0])
plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC曲线 (AUC = {roc_auc:.3f})')
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--', label='随机分类器')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('假正率 (False Positive Rate)', fontsize=12)
plt.ylabel('真正率 (True Positive Rate)', fontsize=12)
plt.title('ROC曲线', fontsize=14, pad=20)
plt.legend(loc="lower right")
plt.grid(True, alpha=0.3)

# 5. 阈值对指标的影响
plt.sca(axes[1])
test_thresholds = np.arange(0.1, 1.0, 0.1)
precisions = []
recalls = []
f1_scores = []

# 演示不同阈值的影响
print(f"\n 不同阈值下的预测结果对比:")
print("阈值 | 精确率 | 召回率 | F1分数")
print("-" * 55)

for thresh in test_thresholds:
    y_pred_thresh = [1 if prob >= thresh else 0 for prob in y_pred_proba]
    precisions.append(precision_score(y_true, y_pred_thresh, zero_division=0))
    recalls.append(recall_score(y_true, y_pred_thresh))
    f1_scores.append(f1_score(y_true, y_pred_thresh, zero_division=0))

    print(f"{thresh:.1f}  | {precisions[-1]:.3f}  | {recalls[-1]:.3f}  | {f1_scores[-1]:.3f}")

plt.plot(test_thresholds, precisions, 'o-', label='精确率', linewidth=2)
plt.plot(test_thresholds, recalls, 's-', label='召回率', linewidth=2)
plt.plot(test_thresholds, f1_scores, '^-', label='F1分数', linewidth=2)
plt.xlabel('分类阈值', fontsize=12)
plt.ylabel('分数', fontsize=12)
plt.title('阈值对指标的影响', fontsize=14, pad=20)
plt.legend()
plt.grid(True, alpha=0.3)

print(f"\n 关键洞察:")
print("• 阈值降低 → 召回率↑，精确率↓（更敏感，更多FP）")
print("• 阈值提高 → 精确率↑，召回率↓（更严格，更多FN）")
print("• 需要根据业务需求选择最佳阈值")

# 6. 性能总结
plt.sca(axes[2])
plt.axis('off')

summary_text = [
    "*** 模型性能总结 ***",
    "",
    f"• 准确率: {accuracy_manual:.3f}",
    f"• 精确率: {precision_manual:.3f}", 
    f"• 召回率: {recall_manual:.3f}",
    f"• F1分数: {f1_manual:.3f}",
    f"• AUC: {roc_auc:.3f}",
    "",
    "*** 分析建议: ***",
    f"• 模型整体准确率: {'良好' if accuracy_manual > 0.7 else '一般'}",
    f"• 精确率: {'较高' if precision_manual > 0.7 else '需提升'}",
    f"• 召回率: {'较高' if recall_manual > 0.7 else '需提升'}",
    f"• 综合性能: {'优秀' if f1_manual > 0.7 else '良好' if f1_manual > 0.5 else '需优化'}"
]

for i, text in enumerate(summary_text):
    plt.text(0.1, 0.9 - i*0.07, text, fontsize=11, 
             verticalalignment='top', transform=plt.gca().transAxes)

plt.tight_layout()
plt.show()

test_thresholds = np.arange(0.1, 1.0, 0.1)

import matplotlib.pyplot as plt
import numpy as np

# 数据
thresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
precisions = [0.500, 0.556, 0.833, 0.833, 0.800, 0.800, 1.000, 1.000, 1.000]
recalls = [1.000, 1.000, 1.000, 1.000, 0.800, 0.800, 0.600, 0.600, 0.200]
f1_scores = [0.667, 0.714, 0.909, 0.909, 0.800, 0.800, 0.750, 0.750, 0.333]

# 可视化分析
plt.figure(figsize=(12, 8))

plt.plot(thresholds, precisions, 'o-', label='精确率', linewidth=3, markersize=8)
plt.plot(thresholds, recalls, 's-', label='召回率', linewidth=3, markersize=8)
plt.plot(thresholds, f1_scores, '^-', label='F1分数', linewidth=3, markersize=10)

# 标记最佳点
best_idx = np.argmax(f1_scores)
plt.axvline(x=thresholds[best_idx], color='red', linestyle='--', alpha=0.7, label=f'最佳阈值 ({thresholds[best_idx]})')
plt.scatter(thresholds[best_idx], f1_scores[best_idx], color='red', s=200, zorder=5)

plt.xlabel('分类阈值', fontsize=12)
plt.ylabel('分数', fontsize=12)
plt.title('阈值性能分析 - 最佳阈值选择', fontsize=14, pad=20)
plt.legend()
plt.grid(True, alpha=0.3)
plt.ylim(0, 1.1)

# 添加数值标注
for i, (t, p, r, f) in enumerate(zip(thresholds, precisions, recalls, f1_scores)):
    plt.annotate(f'F1={f:.3f}', (t, f), textcoords="offset points", xytext=(0,10), ha='center', fontsize=9)

plt.tight_layout()
plt.show()

print(" F1分数排名:")
f1_ranking = sorted(zip(thresholds, f1_scores), key=lambda x: x[1], reverse=True)
for i, (thresh, f1) in enumerate(f1_ranking[:3], 1):
    print(f"第{i}名: 阈值{thresh} → F1分数 = {f1:.3f}")

print("\n 不同业务场景下的最佳选择:")
print("=" * 50)

scenarios = {
    "疾病筛查": {"priority": "召回率", "recommended": [0.3, 0.4], "reason": "不能漏掉任何患者"},
    "垃圾邮件过滤": {"priority": "精确率", "recommended": [0.7, 0.8], "reason": "不能误判重要邮件"}, 
    "平衡场景": {"priority": "F1分数", "recommended": [0.3, 0.4], "reason": "精确率和召回率都很好"},
    "欺诈检测": {"priority": "召回率", "recommended": [0.3, 0.4], "reason": "不能漏掉欺诈交易"}
}

for scenario, info in scenarios.items():
    print(f" {scenario}:")
    print(f"   关注指标: {info['priority']}")
    print(f"   推荐阈值: {info['recommended']}")
    print(f"   原因: {info['reason']}")
    print()

# 创建详细比较表
print(" 前三名阈值详细比较:")
print("阈值 | 精确率 | 召回率 | F1分数 | 优势")
print("-" * 55)

top_thresholds = [(0.3, 0.833, 1.000, 0.909),
                  (0.4, 0.833, 1.000, 0.909), 
                  (0.5, 0.800, 0.800, 0.800)]

for thresh, prec, rec, f1 in top_thresholds:
    if thresh in [0.3, 0.4]:
        advantage = " 完美召回率 + 高精确率"
    else:
        advantage = " 平衡但两者都不突出"
    print(f"{thresh}   | {prec:.3f}   | {rec:.3f}  | {f1:.3f}  | {advantage}")