计算比例Py(Spark)的置信区间

from pyspark import SparkContext
from pyspark.sql import SparkSession
from pyspark.sql.functions import col
from pyspark.sql.functions import stddev_pop

# 创建SparkSession
spark = SparkSession.builder.appName("Confidence Interval").getOrCreate()

# 读取数据集
data = spark.read.csv("data.csv", header=True, inferSchema=True)

# 计算比例Py的估计值
total_count = data.count()
py_count = data.filter(col("event") == 1).count()
py_estimate = py_count / total_count

# 计算置信区间
alpha = 0.05  # 置信水平为95%
z_score = 1.96  # 对应95%置信水平的标准正态分布z值
standard_error = stddev_pop(data["event"]) / total_count**0.5  # 抽样分布的标准误差
lower_bound = py_estimate - z_score * standard_error
upper_bound = py_estimate + z_score * standard_error

# 打印结果
print("Py的置信区间为: [{:.4f}, {:.4f}]".format(lower_bound, upper_bound))