pyspark中的动态条件生成器

from pyspark.sql import SparkSession
from pyspark.sql.functions import col

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 加载数据
data = spark.read.csv("data.csv", header=True, inferSchema=True)

# 定义动态条件生成器函数
def generate_filter(condition):
    if condition == "condition1":
        return col("column1") > 10
    elif condition == "condition2":
        return col("column2") == "value"
    else:
        return col("column3").isNull()

# 根据条件生成过滤器
condition = "condition1"
filter_condition = generate_filter(condition)

# 应用过滤器
filtered_data = data.filter(filter_condition)

# 显示结果
filtered_data.show()