使用日期样本数据填充pyspark dataframe

from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, DateType

# 创建SparkSession
spark = SparkSession.builder.appName("DateSampleDataFrame").getOrCreate()

# 定义Schema
schema = StructType([
    StructField("id", IntegerType(), True),
    StructField("date", DateType(), True)
])

# 创建样本数据
data = [
    (1, "2023-01-01"),
    (2, "2023-02-01"),
    (3, "2023-03-01"),
    (4, "2023-04-01"),
    (5, "2023-05-01")
]

# 创建DataFrame
df = spark.createDataFrame(data, schema)

# 显示DataFrame
df.show()

from pyspark.sql.functions import to_date, lit

# 示例：处理无效日期
df = df.withColumn("date", to_date(df["date"], "yyyy-MM-dd").cast(DateType()))