使用pyspark解析json数据

from pyspark.sql import SparkSession
from pyspark.sql.functions import col

spark = SparkSession.builder.appName("JSONParser").getOrCreate()

df = spark.read.json("path/to/json/file.json")

df.printSchema()
df.show()

# 选择特定的列
df.select("column1", "column2")

# 过滤数据
df.filter(col("column1") > 10)

# 聚合数据
df.groupBy("column1").agg({"column2": "sum"})

# 排序数据
df.orderBy("column1")

# 添加新列
df.withColumn("new_column", col("column1") + col("column2"))

# 保存为Parquet文件
df.write.parquet("path/to/parquet/file.parquet")

# 写入数据库
df.write.format("jdbc").options(url="jdbc:mysql://localhost/db", \
    driver="com.mysql.jdbc.Driver", dbtable="table", user="username", \
    password="password").save()