Pyspark:通过拆分文本来创建新列

from pyspark.sql import SparkSession
from pyspark.sql.functions import split

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例数据
data = [("John Doe", "Software Engineer"),
        ("Jane Smith", "Data Analyst"),
        ("Mike Johnson", "Project Manager")]
df = spark.createDataFrame(data, ["name", "job_title"])

# 使用split()函数拆分job_title列，并创建新列
df = df.withColumn("job_department", split(df.job_title, " ").getItem(0))
df = df.withColumn("job_level", split(df.job_title, " ").getItem(1))

# 显示结果
df.show()

+------------+-----------------+--------------+---------+
|        name|        job_title|job_department|job_level|
+------------+-----------------+--------------+---------+
|    John Doe|Software Engineer|      Software| Engineer|
| Jane Smith|   Data Analyst|          Data| Analyst|
|Mike Johnson| Project Manager|       Project| Manager|
+------------+-----------------+--------------+---------+