如何基于唯一记录和非NA值对pyspark dataframe中的列进行迭代

from pyspark.sql import SparkSession

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例dataframe
data = [("Alice", 25, 160),
        ("Bob", None, 175),
        ("Charlie", 30, None)]
df = spark.createDataFrame(data, ["Name", "Age", "Height"])

# 获取所有列名并迭代
for col_name, col_type in df.dtypes:
    # 过滤掉包含NA值的列
    if col_type != "null":
        # 创建新的dataframe，只包含非NA值的列
        new_df = df.filter(df[col_name].isNotNull()).withColumn(col_name, df[col_name])
        # 打印新的dataframe
        new_df.show()

# 关闭SparkSession
spark.stop()