pandas 大文件操作

import pandas as pd

f = open('./data/ows-raw.txt',encoding='utf-8')
reader = pd.read_table(f,  sep=',',  iterator=True, error_bad_lines=False) #跳过报错行
loop = True
chunkSize = 100000
chunks = []


while loop:
    try:
        chunk = reader.get_chunk(chunkSize)
        chunks.append(chunk)
    except StopIteration:
        loop = False
        print("Iteration is stopped.")


df = pd.concat(chunks, ignore_index=True)

import pandas as pd
import os
import re


target_path = './data/excel/{}.csv'


def dta_to_excel(origin_path):
    CHUNKSIZE = 2000
    reader = pd.read_stata(origin_path, iterator=True)
    file_name = re.sub(r'\.dta', '', origin_path.split('/')[-1])
    print('{} translate start'.format(file_name))

    chunks = []
    while CHUNKSIZE > 0:
        try:
            print('Will get {} lines'.format(CHUNKSIZE))
            chunk = reader.get_chunk(CHUNKSIZE)
            chunks.append(chunk)
        except ValueError:
            print('CHUNKSIZE too large')
            CHUNKSIZE //= 2
    
    df = pd.concat(chunks, ignore_index=True)
    df.to_csv(target_path.format(file_name))
    print('{} translated done'.format(file_name))


if __name__ == '__main__':
    origin_dir = './data/origin'
    #  os.listdir:列出目标路径下的所有文件（文件夹）
    for path in os.listdir(origin_dir):
        dta_to_excel(os.path.join(origin_dir, path))