spark+kafka的使用(单词计数)

kafka的安装参考我的这边博客(https://blog.csdn.net/qq_16563637/article/details/82776276)
1.安装zk集群
2.config/server.properties
添加zk地址:zookeeper.connect=node-1:2181,node-2:2181,node-3:2181
修改broker.id(唯一的)：broker.id=0
3.启动
/bin/kafka-server-start.sh config/server.properties > /dev/null 2>&1 &
4.创建topic
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 3 --topic test
5.列出所有topic
bin/kafka-topics.sh --list --zookeeper localhost:2181
6.向topic中写入数据
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
7.消费数据
bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning
8.查看指定topic的详情
bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
代码如下:

package cn.itcast.spark.day5

import org.apache.spark.storage.StorageLevel
import org.apache.spark.{HashPartitioner, SparkConf}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}

/**
  * Created by root on 2016/5/21.
  */
//(多个zookeeper用,隔开)
//zookeper groupid topics numThreads
//参数:192.168.1.103:2181 g1 test 2
//向kafka中写数据
//bin/kafka-console-producer.sh --broker-list 192.168.1.103:9092 --topic test
object KafkaWordCount {

  val updateFunc = (iter: Iterator[(String, Seq[Int], Option[Int])]) => {
    //iter.flatMap(it=>Some(it._2.sum + it._3.getOrElse(0)).map(x=>(it._1,x)))
    iter.flatMap { case (x, y, z) => Some(y.sum + z.getOrElse(0)).map(i => (x, i)) }
  }


  def main(args: Array[String]) {
    LoggerLevels.setStreamingLogLevels()
    val Array(zkQuorum, group, topics, numThreads) = args
    val sparkConf = new SparkConf().setAppName("KafkaWordCount").setMaster("local[2]")
    val ssc = new StreamingContext(sparkConf, Seconds(5))
    ssc.checkpoint("c://ck2")
    //"alog-2016-04-16,alog-2016-04-17,alog-2016-04-18"
    //"Array((alog-2016-04-16, 2), (alog-2016-04-17, 2), (alog-2016-04-18, 2))"
    val topicMap = topics.split(",").map((_, numThreads.toInt)).toMap
    val data = KafkaUtils.createStream(ssc, zkQuorum, group, topicMap, StorageLevel.MEMORY_AND_DISK_SER)
    val words = data.map(_._2).flatMap(_.split(" "))
    val wordCounts = words.map((_, 1)).updateStateByKey(updateFunc, new HashPartitioner(ssc.sparkContext.defaultParallelism), true)
    wordCounts.print()
    ssc.start()
    ssc.awaitTermination()
  }
}