从pyspark dataframe中的数组列中删除结构

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, expr

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例数据
data = [
    (1, ["a", "b", "c"], {"key1": "value1", "key2": "value2"}),
    (2, ["d", "e", "f"], {"key3": "value3", "key4": "value4"})
]
df = spark.createDataFrame(data, ["id", "array_col", "struct_col"])

# 删除数组列中的结构
df = df.withColumn("array_col_no_struct", expr("transform(array_col, x -> x)"))

# 选择所有其他列和新创建的列
df = df.select([col(c) for c in df.columns if c != "array_col"] + ["array_col_no_struct"])

# 显示结果
df.show()