Spark Streaming DSstream 的updateByKey用法

package org.lm.spark.streaming

import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}

object StatefulWordCountOnLine {
  def main(args: Array[String]): Unit = {

//定义updateByKey使用到的函数
    val updateFunc=(values:Seq[Int],state:Option[Int])=>{
      val currentCount=values.foldLeft(0)(_+_)
      val previousCount=state.getOrElse(0)
      Some(currentCount+previousCount)
    }
    val conf=new SparkConf().setAppName("Stateful Word Count On Line").setMaster("spark://192.168.189.128:7077")
    val sc=new SparkContext(conf)
    val ssc=new StreamingContext(sc,Seconds(5))

//定义持久化存储位置，很重要，必须定义
    ssc.checkpoint("hdfs://192.168.189.128:9000/user/input/StatefulWordCount/log")
    val lines=ssc.socketTextStream("192.168.189.129",9999)
    val words=lines.flatMap(_.split(" "))
    val pairs=words.map(word=>(word,1))

//调用updateStateByKey方法，参数传入前面定义的updateFunc函数

    val statefulDStream=pairs.updateStateByKey[Int](updateFunc)
    statefulDStream.print()
    ssc.start()
    ssc.awaitTermination()
  }

}