移除标点符号在pyspark中不起作用

import pyspark.sql.functions as F

# 假设我们有一个DataFrame，包含一个名为text的列，存储了需要处理的文本数据
df = spark.createDataFrame([(1, "Hello, world!"), (2, "How are you?")], ["id", "text"])

# 使用正则表达式替换标点符号为空字符串
df = df.withColumn("text_without_punctuation", F.regexp_replace("text", "[^\w\s]", ""))

df.show(truncate=False)

+---+--------------+-----------------------+
|id |text          |text_without_punctuation|
+---+--------------+-----------------------+
|1  |Hello, world! |Hello world            |
|2  |How are you?  |How are you            |
+---+--------------+-----------------------+

import pyspark.sql.functions as F

df = spark.createDataFrame([(1, "Hello, world!"), (2, "How are you?")], ["id", "text"])

# 将文本转换为小写，并使用正则表达式替换标点符号为空字符串
df = df.withColumn("text_without_punctuation", F.lower(F.regexp_replace("text", "[^\w\s]", "")))

df.show(truncate=False)

+---+--------------+-----------------------+
|id |text          |text_without_punctuation|
+---+--------------+-----------------------+
|1  |Hello, world! |hello world            |
|2  |How are you?  |how are you            |
+---+--------------+-----------------------+