Spark RDD转换为DataFrame

#构造case class，利用反射机制隐式转换

scala> import spark.implicits._

scala> val rdd= sc.textFile("input/textdata.txt")

scala> case class Person(id:Int,name:String)

scala> val df = rdd.map(_.split(",")).map(x=>Person(x(0).toInt,x(1))).toDF

scala> df.show

+---+--------+

| id| name|

+---+--------+

| 1|zhangsan|

| 2| lisi|

| 3| wangwu|

| 4| zhaoliu|

+---+--------+

 

#通过schema，Row构造dataframe

scala> import org.apache.spark.sql.types._

scala> import org.apache.spark.sql.Row

scala> val structFields = Array(StructField("id",IntegerType,true),StructField("name",StringType,true))

scala> val structType = StructType(structFields) #创建schema结构

scala> val lines= sc.textFile("input/textdata.txt")

scala> val rdd = lines.map(_.split(",")).map(x=>Row(x(0).toInt,x(1))) #创建RDD[Row]

scala> val df = spark.createDataFrame(rdd,structType) #通过RDD[Row],schema构建DataFrame

scala> df.show

+---+--------+

| id| name|

+---+--------+

| 1|zhangsan|

| 2| lisi|

| 3| wangwu|

| 4| zhaoliu|

+---+--------+

 

cat textdata.txt

1,zhangsan

2,lisi

3,wangwu

4,zhaoliu