PySpark中分类值的等价组in ().unique()

from pyspark.sql import SparkSession

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例数据集
data = [("apple", 1), ("banana", 2), ("apple", 3), ("banana", 4), ("orange", 5)]

# 将数据集转换为DataFrame
df = spark.createDataFrame(data, ["fruit", "value"])

# 获取分类值的等价组
equivalent_groups = df.select("fruit").distinct().rdd.flatMap(lambda x: x).collect()

# 打印结果
print(equivalent_groups)

['apple', 'banana', 'orange']