MongoDB的数据清理

-- 注意下面插入的是 new Date("2023-01-01T00:00:00Z") 日期时间类型的，如果插入的是"2023-01-01 00:00:00" 则表示的是字符串类型，而不是时间类型！！
use testdb
db.tb1.insertOne({
    "name": "example1",
    "timestamp": new Date()
})

db.tb1.insertOne({
    "name": "example2",
    "timestamp": new Date("2023-01-01T00:00:00Z")
})

db.tb1.find()

from pymongo import MongoClient
import datetime
import time

MONGO_URI = "mongodb://192.168.31.181:22017"


def clean_old_data_in_batches(
    db_name, collection_name, days_old, batch_size=100, sleep_time=2
):
    client = MongoClient(MONGO_URI)
    db = client[db_name]
    collection = db[collection_name]

    # 计算截止日期
    cutoff_date = str(datetime.datetime.utcnow() - datetime.timedelta(days=days_old))
    print(cutoff_date)

    while True:
        # 查找一小批过期文档
        old_docs = collection.find(
            {"timestamp": {"$lt": cutoff_date}}, limit=batch_size
        )
        print(old_docs)

        old_docs_count = old_docs.count()
        if old_docs_count == 0:
            break

        # 获取要删除文档的ID列表
        old_doc_ids = [doc["_id"] for doc in old_docs]
        # print(old_doc_ids)

        # 删除这一小批文档
        result = collection.delete_many({"_id": {"$in": old_doc_ids}})
        print(f"Deleted {result.deleted_count} documents.")

        # 避免对数据库造成过大压力
        time.sleep(sleep_time)

    client.close()
    print("Batch deletion completed.")


# 删除超过30天的数据，每批次删除1000条，间隔1秒
clean_old_data_in_batches("db1", "tb1", 30, batch_size=1000, sleep_time=1)