199、Spark 2.0之Dataset开发详解-其他常用函数

日期函数：current_date、current_timestamp
数学函数：round
随机函数：rand
字符串函数：concat、concat_ws
自定义udf和udaf函数

文档地址：http://spark.apache.org/docs/latest/api/scala/index.html#org.apache.spark.sql.functions$

代码

object OtherFunction {
  def main(args: Array[String]) {
    val sparkSession = SparkSession
      .builder()
      .appName("OtherFunction")
      .master("local")
      .getOrCreate()

    import sparkSession.implicits._
    import org.apache.spark.sql.functions._

    val employeePath = this.getClass.getClassLoader.getResource("employee.json").getPath
    val departmentPath = this.getClass.getClassLoader.getResource("department.json").getPath

    val employeeDF = sparkSession.read.json(employeePath)
    val departmentDF = sparkSession.read.json(departmentPath)

    val employeeDS = employeeDF.as[Employee]
    val departmentDS = departmentDF.as[Department]

    employeeDS
      .select(employeeDS("name"), current_date(), current_timestamp(), rand(), round(employeeDS("salary"), 2), concat(employeeDS("gender"), employeeDS("age")), concat_ws("|", employeeDS("gender"), employeeDS("age")))
      .show()
  }
}