从大型csv文件C#.Net中删除重复记录

using System;
using System.Collections.Generic;
using System.IO;

class Program
{
    static void Main()
    {
        string inputFile = "input.csv";
        string outputFile = "output.csv";

        // 读取csv文件并加载到内存中
        List<string> records = new List<string>();
        using (StreamReader reader = new StreamReader(inputFile))
        {
            string line;
            while ((line = reader.ReadLine()) != null)
            {
                records.Add(line);
            }
        }

        // 识别和删除重复记录
        HashSet<string> uniqueRecords = new HashSet<string>();
        foreach (string record in records)
        {
            if (!uniqueRecords.Contains(record))
            {
                uniqueRecords.Add(record);
            }
        }

        // 将结果保存到新的csv文件中
        using (StreamWriter writer = new StreamWriter(outputFile))
        {
            foreach (string record in uniqueRecords)
            {
                writer.WriteLine(record);
            }
        }

        Console.WriteLine("重复记录已成功删除并保存到新文件中。");
    }
}