Python Pyspark:使用F.current_date()过滤当前日期之前的1天

from pyspark.sql import SparkSession
from pyspark.sql.functions import current_date, date_sub

# 创建 SparkSession
spark = SparkSession.builder.appName("example").getOrCreate()

# 示例数据
data = [
    ("2023-10-01", "A"),
    ("2023-10-02", "B"),
    ("2023-10-03", "C"),
    ("2023-10-04", "D")
]

columns = ["date_column", "value"]

# 创建 DataFrame
df = spark.createDataFrame(data, columns)

# 过滤当前日期之前的1天的数据
filtered_df = df.filter(date_sub(current_date(), 1) <= df["date_column"])

# 显示结果
filtered_df.show()

from pyspark.sql.functions import to_date

df = df.withColumn("date_column", to_date(df["date_column"], "yyyy-MM-dd"))

from pyspark.sql.functions import current_date, lit

df = df.filter(date_sub(current_date(lit("UTC")), 1) <= df["date_column"])