在集群模式下使用spark将pandas数据帧(.csv)写入本地系统或hdfs

from pyspark.sql import SparkSession
import pandas as pd

spark = SparkSession.builder.appName("WriteDataFrame").getOrCreate()

csv_file = "path/to/input.csv"
pandas_df = pd.read_csv(csv_file)

spark_df = spark.createDataFrame(pandas_df)

output_path = "path/to/output"
output_file = "output.csv"

spark_df.write.csv(output_path + "/" + output_file, header=True)

spark_df.write.format("csv").mode("overwrite").option("header", "true").save(output_path)