Spark -从列读取JSON数组

from pyspark.sql import SparkSession
from pyspark.sql.functions import from_json, col
from pyspark.sql.types import ArrayType, StructType, StructField, StringType

# 创建 SparkSession
spark = SparkSession.builder.appName("Read JSON Array").getOrCreate()

# 示例数据
data = [
    (1, '[{"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}]'),
    (2, '[{"name": "Charlie", "age": 35}]')
]

# 定义 schema
schema = StructType([
    StructField("id", IntegerType(), True),
    StructField("json_array", StringType(), True)
])

# 创建 DataFrame
df = spark.createDataFrame(data, schema)

# 解析 JSON 数组
json_schema = ArrayType(StructType([
    StructField("name", StringType(), True),
    StructField("age", IntegerType(), True)
]))

df_parsed = df.withColumn("parsed_json", from_json(col("json_array"), json_schema))

# 展开 JSON 数组
df_expanded = df_parsed.select("id", "parsed_json.*")

df_expanded.show()