在Pyspark dataframe中拆分输入日志文件

from pyspark.sql import SparkSession
from pyspark.sql.functions import split

spark = SparkSession.builder.appName("LogSplit").getOrCreate()

log_df = spark.read.text("input.log")

split_df = log_df.select(split(log_df.value, " ").alias("log_data"))

expanded_df = split_df.selectExpr("log_data[0] as column1", "log_data[1] as column2", ...)

from pyspark.sql import SparkSession
from pyspark.sql.functions import split

# 创建SparkSession对象
spark = SparkSession.builder.appName("LogSplit").getOrCreate()

# 读取输入日志文件并创建DataFrame
log_df = spark.read.text("input.log")

# 使用split函数拆分日志行
split_df = log_df.select(split(log_df.value, " ").alias("log_data"))

# 展开拆分后的数据
expanded_df = split_df.selectExpr("log_data[0] as column1", "log_data[1] as column2")

# 显示拆分后的数据
expanded_df.show()

# 停止SparkSession
spark.stop()