Spark Streaming整合Kafka，Mysql，实时保存数据到Mysql(基于Receiver的方式)

集群分配如下：

192.168.58.11	spark01
192.168.58.12	spark02
192.168.58.13	spark03
spark版本：spark-2.1.0-bin-hadoop2.7
kafka版本：kafka_2.11-2.0.0

Spark Streaming程序

package com.kk.sparkstreaming.kafka

import org.apache.spark.SparkConf
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.log4j.Level
import org.apache.log4j.Logger
import java.sql.DriverManager
import java.sql.PreparedStatement
import java.sql.Connection

object KafkaReceiver {
   def main(args: Array[String]): Unit = {
     
     // 减少日志输出
     Logger.getLogger("org.apache.spark").setLevel(Level.ERROR)
     Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)
 
     val sparkConf = new SparkConf().setAppName("KafkaReceivers").setMaster("local[2]")
     
     val sparkStreaming = new StreamingContext(sparkConf,Seconds(3))
     
     // 创建topic名称，1表示一次从这个topic中获取一条记录
     val topics = Map("kevin" -> 1)
     
     // 创建Kafka的输入流，制定ZooKeeper地址
     val kafkaStream = KafkaUtils.createStream(sparkStreaming, "192.168.58.11:2181", "mygroup", topics)
     
     // 处理每次接受的数据
     val lineDStream = kafkaStream.map(e => {
        new String(e.toString())
     })
     
     // kafka接收的数据是形式，key为null,取出value
     val logRDD = kafkaStream.map(_._2)
     
     lineDStream.print()
     
     // 对接受的数据进行分词处理
     val datas = logRDD.map(line => {
        // 1,201.105.101.108,http://mystore.jsp/?productid=1,2017020029,2,1 
        val index: Array[String] = line.split(",")
        val ip = index(1);
        (ip, 1)
     })

    // 打印在屏幕
    datas.print()
    
    // 将数据保存在mysql数据库
    datas.foreachRDD(cs => {
      var conn: Connection = null;
      var ps: PreparedStatement = null;
      try {
        Class.forName("com.mysql.jdbc.Driver").newInstance();
        cs.foreachPartition(f => {
          conn = DriverManager.getConnection("jdbc:mysql://192.168.58.14:3306/storm?useUnicode=true&characterEncoding=utf8", "root", "kevin");
          ps = conn.prepareStatement("insert into result values(?,?)");
          f.foreach(s => {
            ps.setString(1, s._1);
            ps.setInt(2, s._2);
            ps.executeUpdate();
          })
        })
      } catch {
        case t: Throwable => t.printStackTrace() // TODO: handle error
      } finally {
        if (ps != null) {
          ps.close()
        }
        if (conn != null) {
          conn.close();
        }
      }
    })
     
    sparkStreaming.start()
    sparkStreaming.awaitTermination()
     
   }
}

pom.xml文件

	UTF-8
	2.2.1
	2.11.1




	
		org.scala-lang
		scala-library
		${scala.version}
	
	
	
		org.scala-lang
		scala-compiler
		${scala.version}
	
	
	
		org.scala-lang
		scala-reflect
		${scala.version}
	
	
	
		org.apache.spark
		spark-core_2.11
		${spark.version}
	
	
	
		org.apache.spark
		spark-streaming_2.11
		${spark.version}
	
	
	
		org.apache.spark
		spark-sql_2.11
		${spark.version}
	
	
	
		org.apache.spark
		spark-streaming-kafka-0-8_2.11
		2.1.1
	

	
		mysql
		mysql-connector-java
		5.1.8
	

测试
1.启动kafka集群，创建消息的生产者
2.将程序提交至集群运行
集群运行出现异常，jar包冲突

Exception in thread "main" java.lang.ClassCastException: kafka.cluster.BrokerEndPoint cannot be cast to kafka.cluster.Broker
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3$$anonfun$apply$6$$anonfun$apply$7.apply(KafkaCluster.scala:90)
	at scala.Option.map(Option.scala:145)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3$$anonfun$apply$6.apply(KafkaCluster.scala:90)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3$$anonfun$apply$6.apply(KafkaCluster.scala:87)
	at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)
	at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)
	at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
	at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
	at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251)
	at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3.apply(KafkaCluster.scala:87)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3.apply(KafkaCluster.scala:86)
	at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)
	at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)
	at scala.collection.immutable.Set$Set1.foreach(Set.scala:74)
	at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251)
	at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2.apply(KafkaCluster.scala:86)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2.apply(KafkaCluster.scala:85)
	at scala.util.Either$RightProjection.flatMap(Either.scala:523)

解决方法
需将kafak的jar包替换成kafka_2.11-0.8.2.1.jar，复制在saprk每个节点的jars目录下