使用PySpark删除表情符号

from pyspark.sql import SparkSession
from pyspark.sql.functions import udf
from pyspark.sql.types import StringType
import re

spark = SparkSession.builder.appName("EmojiRemoval").getOrCreate()

def remove_emoji(text):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)

remove_emoji_udf = udf(remove_emoji, StringType())
spark.udf.register("remove_emoji", remove_emoji_udf)

data = spark.read.text("path/to/input/file.txt")
data = data.withColumn("clean_text", remove_emoji_udf("value"))

data.select("clean_text").write.text("path/to/output/file.txt")