Spark:删除from_json后的空值或仅从json获取值

from pyspark.sql import SparkSession
from pyspark.sql.functions import from_json, col
from pyspark.sql.types import StructType, StructField, StringType

# 创建 SparkSession
spark = SparkSession.builder.appName("example").getOrCreate()

# 示例 JSON 数据
data = [("{}",), ('{"name": "Alice"}',), ('{"age": 30}',)]

# 定义 schema
schema = StructType([
    StructField("name", StringType(), True),
    StructField("age", StringType(), True)
])

# 读取数据并应用 from_json
df = spark.createDataFrame(data, ["value"])
parsed_df = df.select(from_json(col("value"), schema).alias("data")).select("data.*")

# 删除空值
cleaned_df = parsed_df.na.drop()
cleaned_df.show()

from pyspark.sql.functions import get_json_object

# 示例 JSON 数据
data = [('{"name": "Alice", "age": 30}',)]

# 创建 DataFrame
df = spark.createDataFrame(data, ["value"])

# 获取特定值
result_df = df.select(get_json_object(col("value"), "$.name").alias("name"))
result_df.show()