Spark/Scala中包含多个元素的嵌套模式

import org.apache.spark.sql.types._

// 定义嵌套模式
val nestedSchema = StructType(Seq(
  StructField("name", StringType, nullable = false),
  StructField("age", IntegerType, nullable = false),
  StructField("address", StructType(Seq(
    StructField("street", StringType, nullable = false),
    StructField("city", StringType, nullable = false),
    StructField("state", StringType, nullable = false)
  )))
))

// 创建数据集
val data = Seq(
  Row("John", 30, Row("123 Main St", "New York", "NY")),
  Row("Jane", 25, Row("456 Elm St", "San Francisco", "CA"))
)

// 将数据集映射到DataFrame中
val df = spark.createDataFrame(spark.sparkContext.parallelize(data), nestedSchema)

// 查询嵌套字段
df.select("name", "address.city").show()

// 过滤嵌套字段
df.filter("address.state = 'NY'").show()