在PySpark中添加具有滚动最新优先级的列

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, expr, max
from pyspark.sql.window import Window

# 创建SparkSession
spark = SparkSession.builder.appName("Rolling Latest Priority").getOrCreate()

# 创建示例数据
data = [(1, 'A', '2021-01-01'),
        (2, 'B', '2021-01-02'),
        (3, 'C', '2021-01-03'),
        (4, 'A', '2021-01-04'),
        (5, 'B', '2021-01-05'),
        (6, 'C', '2021-01-06'),
        (7, 'A', '2021-01-07')]

df = spark.createDataFrame(data, ['id', 'priority', 'date'])

# 使用窗口函数和滚动最新优先级
windowSpec = Window.partitionBy('priority').orderBy(col('date').cast('timestamp').desc())
df = df.withColumn('latest_priority', max(col('priority')).over(windowSpec))

# 打印结果
df.show()

+---+--------+----------+---------------+
|id |priority|date      |latest_priority|
+---+--------+----------+---------------+
|7  |A       |2021-01-07|A              |
|4  |A       |2021-01-04|A              |
|1  |A       |2021-01-01|A              |
|6  |C       |2021-01-06|C              |
|3  |C       |2021-01-03|C              |
|2  |B       |2021-01-02|B              |
|5  |B       |2021-01-05|B              |
+---+--------+----------+---------------+