Spark编写wordcount程序

package demo

import org.apache.spark.{SparkConf, SparkContext}

//开发一个scala版本的wordcount
/*
bin/spark-submit --master spark://bigdata111:7077 --class demo.MyWordCount /root/temp/mywordcount.jar hdfs://192.168.157.111:9000/input/data.txt hdfs://192.168.157.111:9000/output/0608/wc2
 */
object MyWordCount {
  def main(args: Array[String]): Unit = {
    //配置
    val conf = new SparkConf().setAppName("MyWordCount")  //.setMaster("local")

    //核心创建一个SparkContext
    val sc = new SparkContext(conf)

    //使用sc对象执行任务
//    sc.textFile("hdfs://192.168.157.111:9000/input/data.txt")
//        .flatMap(_.split(" "))
//         .map((_,1))
//          .reduceByKey(_+_)
//            .saveAsTextFile("hdfs://192.168.157.111:9000/output/0608/wc1")
    sc.textFile(args(0))
      .flatMap(_.split(" "))
      .map((_,1))
      .reduceByKey(_+_)
      .saveAsTextFile(args(1))

    //停止SparkContext对象
    sc.stop()
  }
}