Pyspark - RDD提取要聚合的值

from pyspark import SparkContext

# 初始化Spark上下文
sc = SparkContext("local", "Aggregation Example")

# 创建一个键值对RDD
data = [("A", 1), ("B", 2), ("A", 3), ("B", 4), ("C", 5)]
rdd = sc.parallelize(data)

# 使用groupByKey聚合数据
grouped_data = rdd.groupByKey()

# 使用map将grouped_data转换为所需的格式并计算总和
result = grouped_data.map(lambda x: (x[0], sum(x[1]))).collect()

print(result)

[('A', 4), ('B', 6), ('C', 5)]

from pyspark import SparkContext

# 初始化Spark上下文
sc = SparkContext("local", "Aggregation Example")

# 创建一个键值对RDD
data = [("A", 1), ("B", 2), ("A", 3), ("B", 4), ("C", 5)]
rdd = sc.parallelize(data)

# 使用reduceByKey聚合数据
result = rdd.reduceByKey(lambda a, b: a + b).collect()

print(result)

[('A', 4), ('B', 6), ('C', 5)]