数据处理（一）| 从“脏数据”到“干净数据”：数据清洗全流程详细解析与实践指南

# 示例：检查单张图像的有效性
cleaner = DataCleaner()
result = cleaner.check_image_validity("test.jpg")
print("图像检查结果:", result)

# 批量检查图像有效性
image_paths = ["image1.jpg", "image2.jpg", "image3.jpg"]
invalid_images = []
for path in image_paths:
    result = cleaner.check_image_validity(path)
    if not result["is_valid"]:
        invalid_images.append((path, result["error_message"]))

print("无效图像列表:", invalid_images)

图像检查结果: {'is_valid': True, 'error_message': ''}
无效图像列表: [('image2.jpg', '图像尺寸过小'), ('image3.jpg', '图像可能全黑或全白')]

# 示例：查看表格数据的基本信息
data = {
    'A': [1, 2, None, 4],
    'B': [None, 2, 3, 4],
    'value': [1, 2, 100, 3, 4, 5]
}
df = pd.DataFrame(data)

# 查看描述统计
print("描述统计:\n", df.describe())

# 查看字段信息和缺失值
print("\n字段信息:\n")
df.info()

描述统计:
             A    B  value
count  3.0  3.0    6.0
mean   2.333333  3.0   19.0
std    1.527525  1.0   43.108
min    1.0  2.0    1.0
25%    1.5  2.5    2.25
50%    2.0  3.0    3.5
75%    3.0  3.5    4.75
max    4.0  4.0    100.0

字段信息:
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 4 entries, 0 to 3
Data columns (total 3 columns):
 #   Column  Non-Null Count  Dtype  
---  ------  --------------  -----  
 0   A       3 non-null      float64
 1   B       3 non-null      float64
 2   value   4 non-null      int64  
dtypes: float64(2), int64(1)
memory usage: 224.0 bytes

# 示例：填补缺失值
strategy = {'A': 'mean', 'B': 'ffill'}
df_cleaned = cleaner.fix_missing_values(df, strategy)
print("\n处理缺失值后:\n", df_cleaned)

处理缺失值后:
      A    B  value
0  1.0  NaN      1
1  2.0  2.0      2
2  2.333333  3.0    100
3  4.0  4.0      3

# 示例：剔除异常值
df_cleaned = cleaner.remove_outliers(df_cleaned, ['value'], 'zscore', 2.0)
print("\n处理异常值后:\n", df_cleaned)

处理异常值后:
      A    B  value
0  1.0  NaN      1
1  2.0  2.0      2
3  4.0  4.0      3

# 示例：修复图像
repaired_image = cleaner.repair_image("damaged_image.jpg", repair_method="denoise")
cv2.imwrite("repaired_image.jpg", repaired_image)
print("图像修复完成，已保存为 repaired_image.jpg")

# 示例：绘制清洗前后的数据分布
import matplotlib.pyplot as plt

# 清洗前的分布
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.hist(df['value'], bins=10, color='blue', alpha=0.7)
plt.title("清洗前分布")

# 清洗后的分布
plt.subplot(1, 2, 2)
plt.hist(df_cleaned['value'], bins=10, color='green', alpha=0.7)
plt.title("清洗后分布")
plt.show()

# 查看日志文件
with open("cleaning_log.txt", "r", encoding='utf-8') as f:
    print(f.read())

数据清洗日志
==================================================

操作：缺失值处理
详情：应用策略：{'A': 'mean', 'B': 'ffill'}
------------------------------

操作：异常值处理
详情：方法：zscore, 阈值：2.0, 处理列：['value']
------------------------------

操作：图像修复
详情：方法：denoise, 文件：damaged_image.jpg
------------------------------

# 导入必要的库
import os
import cv2
import datetime

# 函数：获取图像文件的基本信息
def get_image_info(image_path):
    # 获取图像文件大小（字节）
    file_size = os.path.getsize(image_path)
    # 读取图像
    image = cv2.imread(image_path)
    # 获取分辨率
    height, width, channels = image.shape
    # 获取文件的时间戳
    timestamp = os.path.getmtime(image_path)
    # 返回图像的基本信息
    return file_size, (width, height), timestamp, image

# 函数：判断图像是否模糊（雾天影像）
def is_blurry(image, threshold=1000):
    # 使用拉普拉斯算子进行模糊度检测
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    variance = cv2.Laplacian(gray, cv2.CV_64F).var()
    return variance < threshold  # 返回是否模糊

# 函数：记录删除或存储待定的图像
def record_action(action, image_path, reason, version_system):
    timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    log_message = f"{timestamp} - {action} image: {image_path}, Reason: {reason}\n"
    # 将记录写入版本管理系统的日志文件
    with open(version_system, 'a') as log_file:
        log_file.write(log_message)

# 函数：处理图像数据
def process_images(image_folder, version_system, discard_threshold=1000):
    for filename in os.listdir(image_folder):
        image_path = os.path.join(image_folder, filename)
        if os.path.isfile(image_path):
            # 获取图像的基本信息
            file_size, resolution, timestamp, image = get_image_info(image_path)
            
            # 检查是否模糊（雾天）
            if is_blurry(image):
                # 判断是否删除或待定
                print(f"Image {filename} is blurry.")
                user_choice = input("Do you want to discard (d) or save to pending (p)?: ").strip().lower()
                
                if user_choice == 'd':
                    # 删除图像并记录
                    os.remove(image_path)
                    record_action("Deleted", image_path, "Foggy image", version_system)
                elif user_choice == 'p':
                    # 将图像保存到待定文件夹并记录
                    pending_folder = os.path.join(image_folder, "pending")
                    if not os.path.exists(pending_folder):
                        os.makedirs(pending_folder)
                    os.rename(image_path, os.path.join(pending_folder, filename))
                    record_action("Moved to pending", image_path, "Foggy image", version_system)

# 主程序
if __name__ == "__main__":
    # 数据集文件夹路径
    image_folder = "path/to/your/images"
    # 数据版本管理日志文件路径
    version_system = "path/to/your/version_log.txt"
    # 处理图像
    process_images(image_folder, version_system)