向PySpark添加S3DistCp

from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from pyspark.sql.types import *
import boto3

spark = SparkSession.builder.appName("S3DistCpExample").getOrCreate()

spark.conf.set("spark.hadoop.fs.s3a.access.key", "your_access_key")
spark.conf.set("spark.hadoop.fs.s3a.secret.key", "your_secret_key")

s3distcp_command = "s3-dist-cp --src s3://source_bucket/path --dest s3://destination_bucket/path"
subprocess.call(s3distcp_command, shell=True)