使用数据帧的Pyspark子选择/子查询连接

from pyspark.sql import SparkSession
from pyspark.sql.functions import col

# 创建SparkSession
spark = SparkSession.builder.appName("SubqueryExample").getOrCreate()

# 创建示例数据帧
data1 = [("Alice", 29), ("Bob", 31), ("Cathy", 25)]
data2 = [("Alice", "New York"), ("Bob", "Chicago"), ("David", "Los Angeles")]

df1 = spark.createDataFrame(data1, ["name", "age"])
df2 = spark.createDataFrame(data2, ["name", "city"])

# 使用子查询进行连接
subquery = df2.filter(col("city") == "New York")
result = df1.join(subquery, on="name", how="inner")

# 显示结果
result.show()

# 使用广播变量
broadcast_df2 = spark.sparkContext.broadcast(df2.collect())
subquery = spark.createDataFrame(broadcast_df2.value).filter(col("city") == "New York")
result = df1.join(subquery, on="name", how="inner")