过拟合不是坏事，是信号：安全攻防中的模型调试艺术

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from sklearn.datasets import make_classification

# 生成模拟安全数据（包含异常样本）
X, y = make_classification(n_samples=1000, n_classes=2, weights=[0.95, 0.05], random_state=42)

# 插入一些异常样本
n_anomalies = 50
X_anomalies = np.random.rand(n_anomalies, X.shape[1]) * 10  # 生成远离正常分布的异常样本
y_anomalies = np.random.randint(0, 2, n_anomalies)  # 随机标注

# 合并数据
X = np.vstack([X, X_anomalies])
y = np.concatenate([y, y_anomalies])

# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 1. 正常训练（适中复杂度）
model_normal = RandomForestClassifier(n_estimators=100, max_depth=10, random_state=42)
model_normal.fit(X_train, y_train)

# 2. 过拟合训练（高复杂度）
model_overfit = RandomForestClassifier(n_estimators=500, max_depth=None, min_samples_split=2, min_samples_leaf=1, random_state=42)
model_overfit.fit(X_train, y_train)

# 计算训练集和测试集分数
def calculate_scores(model, X_train, y_train, X_test, y_test):
    train_score = f1_score(y_train, model.predict(X_train))
    test_score = f1_score(y_test, model.predict(X_test))
    return train_score, test_score

normal_train, normal_test = calculate_scores(model_normal, X_train, y_train, X_test, y_test)
overfit_train, overfit_test = calculate_scores(model_overfit, X_train, y_train, X_test, y_test)

print(f"正常模型 - 训练集F1: {normal_train:.4f}, 测试集F1: {normal_test:.4f}")
print(f"过拟合模型 - 训练集F1: {overfit_train:.4f}, 测试集F1: {overfit_test:.4f}")

# 3. 检测过拟合样本
def detect_overfit_samples(model, X_train, y_train):
    """检测模型过度拟合的样本"""
    # 获取样本预测概率
    y_pred_proba = model.predict_proba(X_train)
    
    # 计算样本的预测置信度
    confidence = np.max(y_pred_proba, axis=1)
    
    # 计算样本的预测正确性
    y_pred = model.predict(X_train)
    is_correct = (y_pred == y_train)
    
    # 找出模型高度自信但预测错误的样本（可能是异常样本）
    overfit_samples = np.where((confidence > 0.95) & (~is_correct))[0]
    
    return overfit_samples, confidence, is_correct

# 检测过拟合样本
normal_overfit_samples, normal_confidence, normal_is_correct = detect_overfit_samples(model_normal, X_train, y_train)
overfit_overfit_samples, overfit_confidence, overfit_is_correct = detect_overfit_samples(model_overfit, X_train, y_train)

print(f"\n正常模型检测到的过拟合样本数: {len(normal_overfit_samples)}")
print(f"过拟合模型检测到的过拟合样本数: {len(overfit_overfit_samples)}")

# 4. 分析过拟合样本的特征
def analyze_overfit_samples(X_train, y_train, overfit_samples, original_data_size=1000):
    """分析过拟合样本的特征"""
    # 区分原始样本和异常样本
    is_original = np.arange(len(X_train)) < original_data_size
    
    # 计算过拟合样本中异常样本的比例
    overfit_original = np.sum(is_original[overfit_samples])
    overfit_anomalies = len(overfit_samples) - overfit_original
    
    print(f"\n过拟合样本分析:")
    print(f"过拟合样本总数: {len(overfit_samples)}")
    print(f"其中原始样本数: {overfit_original}")
    print(f"其中异常样本数: {overfit_anomalies}")
    print(f"异常样本占比: {overfit_anomalies / len(overfit_samples):.2%}")
    
    return overfit_original, overfit_anomalies

# 分析过拟合样本
analyze_overfit_samples(X_train, y_train, normal_overfit_samples)
analyze_overfit_samples(X_train, y_train, overfit_overfit_samples)

# 5. 可视化过拟合情况
plt.figure(figsize=(12, 6))

# 绘制置信度分布图
plt.subplot(1, 2, 1)
plt.hist(normal_confidence, bins=50, alpha=0.5, label='Normal Model')
plt.hist(overfit_confidence, bins=50, alpha=0.5, label='Overfit Model')
plt.xlabel('Prediction Confidence')
plt.ylabel('Number of Samples')
plt.title('Prediction Confidence Distribution')
plt.legend()

# 绘制过拟合样本的特征分布
plt.subplot(1, 2, 2)
if len(overfit_overfit_samples) > 0:
    # 选择两个重要特征进行可视化
    feature1 = 0
    feature2 = 1
    
    # 绘制所有样本
    plt.scatter(X_train[:, feature1], X_train[:, feature2], c=y_train, cmap='coolwarm', alpha=0.3, label='All Samples')
    
    # 绘制过拟合样本
    plt.scatter(X_train[overfit_overfit_samples, feature1], X_train[overfit_overfit_samples, feature2], 
                c='black', marker='x', s=100, label='Overfit Samples')
    
    plt.xlabel(f'Feature {feature1}')
    plt.ylabel(f'Feature {feature2}')
    plt.title('Overfit Samples Visualization')
    plt.legend()

plt.tight_layout()
plt.savefig('overfit_analysis.png')
print("\n过拟合分析可视化完成，保存为overfit_analysis.png")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score

# 生成安全相关的分类数据（两个类别，非线性可分）
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, n_redundant=5, n_classes=2, weights=[0.9, 0.1], random_state=42)

# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 1. 训练不同复杂度的模型
models = {
    "Underfit (低复杂度)": SVC(kernel='linear', C=0.1, random_state=42),
    "Normal (适中复杂度)": SVC(kernel='rbf', C=1.0, gamma='scale', random_state=42),
    "Overfit (高复杂度)": SVC(kernel='rbf', C=100.0, gamma=0.1, random_state=42)
}

# 训练模型并计算分数
train_scores = {}
test_scores = {}
models_fitted = {}

for name, model in models.items():
    model.fit(X_train, y_train)
    models_fitted[name] = model
    train_scores[name] = f1_score(y_train, model.predict(X_train))
    test_scores[name] = f1_score(y_test, model.predict(X_test))
    print(f"{name} - 训练集F1: {train_scores[name]:.4f}, 测试集F1: {test_scores[name]:.4f}")

# 2. 分析模型在不同样本上的表现
def analyze_sample_performance(model, X, y, name):
    """分析模型在不同样本上的表现"""
    y_pred = model.predict(X)
    y_pred_proba = model.decision_function(X)
    
    # 计算样本级别的性能指标
    is_correct = (y_pred == y)
    confidence = np.abs(y_pred_proba)
    
    # 按类别分析
    for cls in [0, 1]:
        cls_idx = (y == cls)
        cls_correct = is_correct[cls_idx]
        cls_confidence = confidence[cls_idx]
        
        print(f"\n{name} - 类别 {cls}:")
        print(f"  样本数: {len(cls_idx)}")
        print(f"  正确率: {np.mean(cls_correct):.2%}")
        print(f"  平均置信度: {np.mean(cls_confidence):.4f}")
        print(f"  置信度标准差: {np.std(cls_confidence):.4f}")
    
    return is_correct, confidence

# 分析不同模型的样本表现
print("\n=== 模型样本表现分析 ===")
for name, model in models_fitted.items():
    print(f"\n--- {name} ---")
    analyze_sample_performance(model, X_test, y_test, name)

# 3. 识别模型的弱点样本
def identify_weakness_samples(model, X, y):
    """识别模型的弱点样本"""
    y_pred = model.predict(X)
    y_pred_proba = model.decision_function(X)
    
    # 找出模型错误分类且置信度高的样本（模型认为自己是对的，但实际是错的）
    is_incorrect = (y_pred != y)
    confidence = np.abs(y_pred_proba)
    
    # 弱点样本：错误分类且置信度高于阈值
    weakness_samples = np.where((is_incorrect) & (confidence > np.mean(confidence)))[0]
    
    return weakness_samples, is_incorrect, confidence

# 识别不同模型的弱点样本
weakness_samples = {}

for name, model in models_fitted.items():
    ws, is_incorrect, confidence = identify_weakness_samples(model, X_test, y_test)
    weakness_samples[name] = ws
    print(f"\n{name} - 弱点样本数: {len(ws)}, 占测试集比例: {len(ws)/len(X_test):.2%}")

# 4. 分析弱点样本的特征重要性
def analyze_weakness_features(X, weakness_samples):
    """分析弱点样本的特征重要性"""
    if len(weakness_samples) == 0:
        return None
    
    # 计算弱点样本与正常样本的特征差异
    X_weak = X[weakness_samples]
    X_normal = np.delete(X, weakness_samples, axis=0)
    
    # 计算每个特征的均值差异
    feature_diff = np.mean(X_weak, axis=0) - np.mean(X_normal, axis=0)
    feature_importance = np.abs(feature_diff)
    
    # 找出最重要的特征
    top_features = np.argsort(feature_importance)[::-1][:5]
    
    print(f"\n弱点样本的重要特征（前5个）:")
    for i, idx in enumerate(top_features):
        print(f"  特征 {idx}: 差异={feature_diff[idx]:.4f}, 重要性={feature_importance[idx]:.4f}")
    
    return feature_importance

# 分析过拟合模型的弱点样本
print("\n=== 过拟合模型弱点分析 ===")
overfit_weakness = weakness_samples["Overfit (高复杂度)"]
feature_importance = analyze_weakness_features(X_test, overfit_weakness)

# 5. 可视化不同模型的决策边界（仅使用前两个特征进行可视化）
def plot_decision_boundary(model, X, y, name, ax):
    """绘制模型的决策边界"""
    h = .02  # 网格步长
    
    # 创建网格
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    
    # 仅使用前两个特征进行预测
    model_2d = SVC(kernel=model.kernel, C=model.C, gamma=model.gamma if hasattr(model, 'gamma') else 'scale', random_state=42)
    model_2d.fit(X[:, :2], y)
    
    # 预测网格点
    Z = model_2d.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    
    # 绘制决策边界
    ax.contourf(xx, yy, Z, alpha=0.3, cmap='coolwarm')
    ax.scatter(X[:, 0], X[:, 1], c=y, cmap='coolwarm', edgecolors='k')
    ax.set_title(name)
    ax.set_xlabel('Feature 0')
    ax.set_ylabel('Feature 1')

# 绘制决策边界
fig, axes = plt.subplots(1, 3, figsize=(18, 6))

for i, (name, model) in enumerate(models_fitted.items()):
    plot_decision_boundary(model, X_train[:, :2], y_train, name, axes[i])

plt.tight_layout()
plt.savefig('decision_boundaries.png')
print("\n决策边界可视化完成，保存为decision_boundaries.png")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score

# 生成安全相关的分类数据
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, n_redundant=5, n_classes=2, weights=[0.9, 0.1], random_state=42)

# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 1. 训练一个过拟合模型
model_overfit = RandomForestClassifier(n_estimators=500, max_depth=None, random_state=42)
model_overfit.fit(X_train, y_train)

# 2. 使用过拟合模型生成对抗样本
def generate_adversarial_samples(model, X, y, epsilon=0.1, n_samples=50):
    """使用过拟合模型生成对抗样本"""
    adversarial_samples = []
    original_samples = []
    original_labels = []
    
    # 选择少数类样本进行攻击（安全场景下通常关注少数类攻击）
    minority_idx = np.where(y == 1)[0]
    selected_idx = np.random.choice(minority_idx, min(n_samples, len(minority_idx)), replace=False)
    
    for idx in selected_idx:
        x_original = X[idx]
        y_original = y[idx]
        
        # 生成对抗样本：在特征空间中向模型决策边界相反方向移动
        # 这里使用简单的梯度近似方法
        x_adversarial = x_original.copy()
        
        # 计算每个特征的重要性（使用随机森林的特征重要性）
        feature_importance = model.feature_importances_
        
        # 对重要特征添加扰动，使模型误分类
        for i in range(len(x_adversarial)):
            if feature_importance[i] > np.mean(feature_importance):
                # 向使模型预测概率降低的方向添加扰动
                # 对于随机森林，我们使用简单的启发式方法
                x_adversarial[i] += epsilon * np.sign(np.random.randn())
        
        adversarial_samples.append(x_adversarial)
        original_samples.append(x_original)
        original_labels.append(y_original)
    
    return np.array(adversarial_samples), np.array(original_samples), np.array(original_labels)

# 生成对抗样本
adversarial_samples, original_samples, original_labels = generate_adversarial_samples(model_overfit, X_test, y_test, epsilon=0.5, n_samples=50)

# 3. 测试对抗样本对不同模型的影响
# 训练不同复杂度的模型
models = {
    "Underfit": RandomForestClassifier(n_estimators=10, max_depth=5, random_state=42),
    "Normal": RandomForestClassifier(n_estimators=100, max_depth=20, random_state=42),
    "Overfit": RandomForestClassifier(n_estimators=500, max_depth=None, random_state=42)
}

# 训练模型
for name, model in models.items():
    model.fit(X_train, y_train)

# 测试对抗样本
print("=== 对抗样本测试 ===")

for name, model in models.items():
    # 测试原始样本
    original_pred = model.predict(original_samples)
    original_acc = f1_score(original_labels, original_pred)
    
    # 测试对抗样本
    adversarial_pred = model.predict(adversarial_samples)
    adversarial_acc = f1_score(original_labels, adversarial_pred)
    
    # 计算对抗样本的成功率（模型误分类的比例）
    attack_success = np.mean(adversarial_pred != original_labels)
    
    print(f"\n{name} 模型:")
    print(f"  原始样本F1分数: {original_acc:.4f}")
    print(f"  对抗样本F1分数: {adversarial_acc:.4f}")
    print(f"  攻击成功率: {attack_success:.2%}")

# 4. 分析对抗样本的特征变化
def analyze_adversarial_features(original_samples, adversarial_samples, feature_importance):
    """分析对抗样本的特征变化"""
    # 计算特征变化量
    feature_changes = np.mean(np.abs(adversarial_samples - original_samples), axis=0)
    
    # 按特征重要性排序
    sorted_idx = np.argsort(feature_importance)[::-1]
    
    print(f"\n=== 对抗样本特征变化分析 ===")
    print(f"前10个重要特征的平均变化:")
    
    for i in range(min(10, len(sorted_idx))):
        idx = sorted_idx[i]
        print(f"  特征 {idx}: 重要性={feature_importance[idx]:.4f}, 平均变化={feature_changes[idx]:.4f}")
    
    # 计算特征重要性与变化量的相关性
    correlation = np.corrcoef(feature_importance, feature_changes)[0, 1]
    print(f"\n特征重要性与变化量的相关性: {correlation:.4f}")

# 分析对抗样本特征变化
analyze_adversarial_features(original_samples, adversarial_samples, model_overfit.feature_importances_)

# 5. 可视化对抗样本分布
plt.figure(figsize=(12, 6))

# 绘制前两个特征的分布
plt.subplot(1, 2, 1)
plt.scatter(original_samples[:, 0], original_samples[:, 1], c='blue', label='Original Samples', alpha=0.7)
plt.scatter(adversarial_samples[:, 0], adversarial_samples[:, 1], c='red', label='Adversarial Samples', alpha=0.7)
plt.xlabel('Feature 0')
plt.ylabel('Feature 1')
plt.title('Original vs Adversarial Samples (Feature 0 vs Feature 1)')
plt.legend()

# 绘制特征变化直方图
plt.subplot(1, 2, 2)
feature_changes = np.mean(np.abs(adversarial_samples - original_samples), axis=0)
plt.bar(range(len(feature_changes)), feature_changes)
plt.xlabel('Feature Index')
plt.ylabel('Average Absolute Change')
plt.title('Average Feature Change in Adversarial Samples')

plt.tight_layout()
plt.savefig('adversarial_samples_analysis.png')
print("\n对抗样本分析可视化完成，保存为adversarial_samples_analysis.png")

import numpy as np
from sklearn.metrics import f1_score
from sklearn.model_selection import train_test_split

def train_models_with_different_complexity(base_model, X_train, y_train, complexity_params):
    """训练不同复杂度的模型"""
    models = {}
    
    for name, params in complexity_params.items():
        model = base_model(**params)
        model.fit(X_train, y_train)
        models[name] = model
    
    return models

def calculate_overfit_metrics(models, X_train, y_train, X_test, y_test, scoring='f1'):
    """计算不同模型的过拟合指标"""
    metrics = {}
    
    for name, model in models.items():
        train_pred = model.predict(X_train)
        test_pred = model.predict(X_test)
        
        if scoring == 'f1':
            train_score = f1_score(y_train, train_pred)
            test_score = f1_score(y_test, test_pred)
        else:
            raise ValueError(f"不支持的评估指标: {scoring}")
        
        # 计算过拟合程度
        overfit_degree = train_score - test_score
        
        metrics[name] = {
            'train_score': train_score,
            'test_score': test_score,
            'overfit_degree': overfit_degree
        }
    
    return metrics

def detect_overfit_samples(model, X, y, confidence_threshold=0.95):
    """检测模型过度拟合的样本"""
    # 获取模型预测概率
    y_pred_proba = model.predict_proba(X)
    y_pred = model.predict(X)
    
    # 计算预测置信度
    confidence = np.max(y_pred_proba, axis=1)
    
    # 找出模型高度自信但预测错误的样本
    overfit_samples = np.where((confidence > confidence_threshold) & (y_pred != y))[0]
    
    return overfit_samples, confidence, y_pred

def analyze_overfit_patterns(model, X, y, overfit_samples):
    """分析过拟合样本的模式"""
    if len(overfit_samples) == 0:
        return None
    
    # 计算过拟合样本和正常样本的特征差异
    X_overfit = X[overfit_samples]
    X_normal = np.delete(X, overfit_samples, axis=0)
    
    # 计算特征均值差异
    feature_diff = np.mean(X_overfit, axis=0) - np.mean(X_normal, axis=0)
    
    # 计算特征重要性（如果模型支持）
    feature_importance = None
    if hasattr(model, 'feature_importances_'):
        feature_importance = model.feature_importances_
    elif hasattr(model, 'coef_'):
        feature_importance = np.abs(model.coef_[0])
    
    return {
        'feature_diff': feature_diff,
        'feature_importance': feature_importance,
        'overfit_samples_count': len(overfit_samples),
        'overfit_samples_ratio': len(overfit_samples) / len(X)
    }

def generate_adversarial_samples_from_overfit(model, X, y, epsilon=0.1, n_samples=50):
    """从过拟合模型生成对抗样本"""
    # 实现简单的对抗样本生成逻辑
    # 这里使用随机扰动方法，实际应用中可以使用更高级的方法
    adversarial_samples = []
    
    # 选择少数类样本
    minority_idx = np.where(y == 1)[0]
    selected_idx = np.random.choice(minority_idx, min(n_samples, len(minority_idx)), replace=False)
    
    for idx in selected_idx:
        x_original = X[idx]
        # 添加随机扰动
        x_adversarial = x_original + epsilon * np.random.randn(len(x_original))
        adversarial_samples.append(x_adversarial)
    
    return np.array(adversarial_samples)