Spark嵌套复杂数据帧

from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, IntegerType, ArrayType, MapType, StringType

# 创建 SparkSession
spark = SparkSession.builder.appName("NestedDataFrameExample").getOrCreate()

# 定义嵌套结构
schema = StructType([
    StructField("id", IntegerType(), True),
    StructField("values", ArrayType(IntegerType()), True),
    StructField("info", MapType(StringType(), StringType()), True)
])

# 创建数据
data = [
    (1, [1, 2, 3], {"key1": "value1", "key2": "value2"}),
    (2, [4, 5], {"key3": "value3"})
]

# 创建 DataFrame
df = spark.createDataFrame(data, schema)

# 显示 DataFrame
df.show()

# 查询嵌套数组
df.select("id", "values[0]").show()

# 查询嵌套映射
df.select("id", "info['key1']").show()

# 使用 explode 函数展开嵌套数组
from pyspark.sql.functions import explode

exploded_df = df.select("id", explode("values").alias("value"))
exploded_df.show()

from pyspark.sql.functions import col, when, lit

# 处理嵌套数组中的空值
df = df.na.fill(value=lit([]), subset=["values"])

# 处理嵌套映射中的空值
df = df.na.fill(value=lit({}), subset=["info"])