当数据包含具有两个不同DataTypes的嵌套数组时，在PySpark中定义模式

{
  "id": 1,
  "values": [
    {"name": "Alice", "age": 30},
    {"name": "Bob", "age": 25}
  ]
}

from pyspark.sql.types import StructType, StructField, StringType, IntegerType, ArrayType

# 定义嵌套的结构类型
nested_schema = StructType([
    StructField("name", StringType(), True),
    StructField("age", IntegerType(), True)
])

# 定义外层的结构类型
schema = StructType([
    StructField("id", IntegerType(), True),
    StructField("values", ArrayType(nested_schema), True)
])

from pyspark.sql import SparkSession

# 创建SparkSession
spark = SparkSession.builder.appName("NestedArrayExample").getOrCreate()

# 定义Schema
nested_schema = StructType([
    StructField("name", StringType(), True),
    StructField("age", IntegerType(), True)
])

schema = StructType([
    StructField("id", IntegerType(), True),
    StructField("values", ArrayType(nested_schema), True)
])

# 读取数据
data = [
    (1, [{"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}]),
    (2, [{"name": "Charlie", "age": 35}, {"name": "David", "age": 40}])
]

df = spark.createDataFrame(data, schema)

# 显示数据
df.show(truncate=False)