本地模式使用JAVA SACLA 开发 Spark SQL DataFrame

原文件：

{"name":"Michael"}
{"name":"Andy", "age":30}
{"name":"Justin", "age":19}

java

 

package com.dt.sparkApps.sql;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;

public class DataFrameOps {

 public static void main(String[] args) {
  // TODO Auto-generated method stub
  
  SparkConf conf=new SparkConf().setAppName("DataFrameOps").setMaster("local");
  
  JavaSparkContext sc=new JavaSparkContext(conf);
  
  SQLContext sqlContext=new SQLContext(sc);
  
  //DataFrame df=sqlContext.read().json("hdfs://master:9000/library/people.json");
  DataFrame df=sqlContext.read().json("G://IMFBigDataSpark2016//tesdata//people.json");
  
  df.show();
  
  df.printSchema();
  
  df.select("name").show();
  //select name,age+10 fom table;
  df.select(df.col("name"),df.col("age").plus(10)).show();
     //select * from table whee age>10
     df.filter(df.col("age").gt(10)).show();
    
     //select count(1) from table groupby age;
     df.groupBy(df.col("age")).count().show();
 
    
 }

}
结果

 

scala

package com.dt.spark.sql

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext

object DataFrameOps {
     def main(args: Array[String]){
 
      val conf = new SparkConf() //创建SparkConf对象
      conf.setAppName("Wow,My First Spark DataFrame App!") //设置应用程序的名称，在程序运行的监控界面可以看到名称
      conf.setMaster("local") //此时，程序在本地运行，不需要安装Spark集群
     
      val sc = new SparkContext(conf)
      val sqlContext =new SQLContext(sc)
     
    //  val df =sqlContext.read.json("hdfs://master:9000/library/people.json");
    val df =sqlContext.read.json("G://IMFBigDataSpark2016//tesdata//people.json");
  
     
     
      df.show()
      df.printSchema()
      df.select("name").show()
      df.select(df("name"), df("age")+10).show()
     
      df.filter(df("age")>10).show()
     
     
     
     }
    
}