“脏”数据毁一生？教你用大数据清洗术，给数据洗个“澡”！

import pandas as pd

df = pd.read_csv("users.csv")

# 查看缺失情况
print(df.isnull().sum())

# 处理方式1：填充默认值
df['phone'] = df['phone'].fillna('未知')

# 处理方式2：用均值/中位数填充
df['age'] = df['age'].fillna(df['age'].median())

# 处理方式3：直接删除
df = df.dropna(subset=['email'])  # 邮箱必须有

# 时间格式统一
df['register_date'] = pd.to_datetime(df['register_date'], errors='coerce')

# 字段标准化：全转小写
df['city'] = df['city'].str.lower().str.strip()

# 找出年龄不合理的数据
df = df[(df['age'] >= 0) & (df['age'] <= 120)]

df = df.drop_duplicates(subset=['user_id'], keep='first')

{
  "user_id": "U1001",
  "action": "click",
  "timestamp": "07/01/2025 10:00:00",
  "city": " SH ",
  "age": -1,
  "device": null
}

import pandas as pd

df = pd.read_json("user_logs.json", lines=True)

# 1. 统一时间格式
df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')

# 2. 去除非法年龄
df = df[(df['age'] >= 0) & (df['age'] <= 100)]

# 3. 去除前后空格+大小写统一
df['city'] = df['city'].str.strip().str.lower()

# 4. 缺失设备填充默认
df['device'] = df['device'].fillna("unknown")

# 5. 重复记录去重
df = df.drop_duplicates()