检测并保存pandas数据帧中的重复行

import pandas as pd

# 创建一个示例数据帧
data = {
    'A': [1, 2, 2, 3, 4, 4],
    'B': ['a', 'b', 'b', 'c', 'd', 'd']
}
df = pd.DataFrame(data)

# 检测重复行
duplicates = df[df.duplicated(keep=False)]

# 保存重复行到新的数据帧
duplicates_df = df[df.duplicated(keep=False)]

# 打印重复行
print("重复行:")
print(duplicates_df)

# 保存到 CSV 文件（可选）
duplicates_df.to_csv('duplicates.csv', index=False)

partial_duplicates = df[df.duplicated(subset=['A'], keep=False)]
print("部分重复行:")
print(partial_duplicates)

df_no_duplicates = df.drop_duplicates(keep='first')
print("去除重复行后的数据帧:")
print(df_no_duplicates)

import dask.dataframe as dd

ddf = dd.from_pandas(df, npartitions=2)
duplicates_ddf = ddf[ddf.duplicated(keep=False)].compute()
print("大规模数据的重复行:")
print(duplicates_ddf)