对照主表单列spark.sql验证2列中的数据

val spark = SparkSession.builder()
  .appName("Data Validation")
  .master("local")
  .getOrCreate()

val mainDF = spark.read.format("csv").option("header", "true").load("path/to/main_table.csv")
val validationDF = spark.read.format("csv").option("header", "true").load("path/to/validation_table.csv")

mainDF.createOrReplaceTempView("main_table")
validationDF.createOrReplaceTempView("validation_table")

val query = """
  SELECT main_table.column1, main_table.column2
  FROM main_table
  LEFT JOIN validation_table
  ON main_table.column1 = validation_table.column1
  WHERE main_table.column2 != validation_table.column2 OR validation_table.column1 IS NULL
"""
val result = spark.sql(query)

result.show()