使用Python3删除MongoDB中重复数据的方法

pip install pymongo

from pymongo import MongoClient

# 创建MongoDB连接
client = MongoClient('mongodb://localhost:27017/')

# 选择数据库和集合
db = client['database_name']
collection = db['collection_name']

pipeline = [
    {
        "$group": {
            "_id": {
                "field1": "$field1",
                "field2": "$field2"
            },
            "duplicates": {"$addToSet": "$_id"},
            "count": {"$sum": 1}
        }
    },
    {
        "$match": {
            "count": {"$gt": 1}
        }
    }
]

duplicate_data = list(collection.aggregate(pipeline))

for data in duplicate_data:
    ids_to_remove = data['duplicates']
    collection.update_many({"_id": {"$in": ids_to_remove}}, {"$pull": {"_id": {"$in": ids_to_remove}}})