05-快速理解SparkSQL的DataSet

package com.javaedge.bigdata.cp04

import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}

object DatasetApp {

  def main(args: Array[String]): Unit = {
    val projectRootPath = "/Users/javaedge/Downloads/soft/sparksql-train"
    val spark = SparkSession.builder()
      .master("local").appName("DatasetApp")
      .getOrCreate()
    import spark.implicits._

    // 创建一个包含一条记录的Seq，这条记录包含一个名为 "JavaEdge" 年龄为 18 的人员信息
    val ds: Dataset[Person] = Seq(Person("JavaEdge", "18"))
      // 将Seq转换为一个Dataset[Person]类型数据集，该数据集只包含一条记录
      .toDS()
    ds.show()

    val primitiveDS: Dataset[Int] = Seq(1, 2, 3).toDS()
    primitiveDS.map(x => x + 1).collect().foreach(println)

    val peopleDF: DataFrame = spark.read.json(projectRootPath + "/data/people.json")
    val peopleDS: Dataset[Person] = peopleDF.as[Person]
    peopleDS.show(false)
    peopleDF.select("name").show()
    peopleDS.map(x => x.name).show()

    spark.stop()
  }

  /**
   * 自定义的 case class，其中包含两个属性
   */
  private case class Person(name: String, age: String)

}

output：
+--------+---+
|    name|age|
+--------+---+
|JavaEdge| 18|
+--------+---+

2
3
4
+----+-------+
|age |name   |
+----+-------+
|null|Michael|
|30  |Andy   |
|19  |Justin |
+----+-------+

+-------+
|   name|
+-------+
|Michael|
|   Andy|
| Justin|
+-------+

+-------+
|  value|
+-------+
|Michael|
|   Andy|
| Justin|
+-------+

val peopleDF: DataFrame = spark.read.json(projectRootPath + "/data/people.json")
val peopleDS: Dataset[Person] = peopleDF.as[Person]
peopleDS.show(false)

// 弱语言类型，运行时才报错
peopleDF.select("nameEdge").show()