使用PySpark从Spark数据帧中的groupby结果创建时间序列

from pyspark.sql import SparkSession
from pyspark.sql.functions import col
from pyspark.sql.window import Window
from pyspark.sql.functions import lag
from pyspark.sql.functions import when
from pyspark.sql.functions import sum

spark = SparkSession.builder.appName("TimeSeries").getOrCreate()

df = spark.read.format("csv").option("header", "true").load("data.csv")

grouped_df = df.groupby("time").agg(sum("value").alias("sum_value")).orderBy("time")

windowSpec = Window.orderBy("time")
time_series_df = grouped_df.select("time", "sum_value", lag("sum_value").over(windowSpec).alias("prev_sum_value"))

time_series_df = time_series_df.withColumn("diff", when(col("prev_sum_value").isNull(), 0).otherwise(col("sum_value") - col("prev_sum_value")))