SparkStreaming的单词计数案例

import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
  * 需要用到nc -lk 9999
  * */
/**
  * Created by Administrator on 2018/5/11.
  */
object SparkStreaming1 {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("workcount").setMaster("local[2]")
    //因为只有一个线程
    //无法做到一边接收数据一边做transiformation处理
    val sc = new SparkContext(conf)
    val scc = new StreamingContext(sc,Seconds(2))
    //创建StreamingContext程序入口
    val dstream: ReceiverInputDStream[String] = scc.socketTextStream("hadoop02",9999)
    //监听虚拟机端口发送的消息
    /**
      * 进行单词计数
      * */
    dstream.
      flatMap(line=>line.split(" ")).
      map(x=>(x,1)).
      reduceByKey(_+_).print()
    /**
      * print操作类似collect
      * */
    scc.start()
    scc.awaitTermination()
    scc.stop()
    /**
      * 后面三个固定搭配
      * */
  }
}