如何在PySpark中将十六进制字符串转换为十六进制字节格式

import binascii
from pyspark.sql.functions import udf
from pyspark.sql.types import BinaryType

# 定义一个UDF函数，用于将十六进制字符串转换为十六进制字节格式
hex_to_bytes_udf = udf(lambda hex_str: binascii.unhexlify(hex_str), BinaryType())

# 假设有一个DataFrame，包含一个名为hex_str的列，存储了十六进制字符串
df = spark.createDataFrame([(1, "48656c6c6f20576f726c64"), (2, "5468697320697320612074657374")], ["id", "hex_str"])

# 使用UDF函数将hex_str列转换为bytes列
df = df.withColumn("bytes", hex_to_bytes_udf(df.hex_str))

# 打印转换结果
df.show(truncate=False)

+---+----------------+------------------+
|id |hex_str         |bytes             |
+---+----------------+------------------+
|1  |48656c6c6f20576f726c64|Hello World       |
|2  |5468697320697320612074657374|This is a test    |
+---+----------------+------------------+