Spark Streaming整合Kafka，Mysql，实时保存数据到Mysql(直接读取方式)

集群分配如下：

192.168.58.11	spark01
192.168.58.12	spark02
192.168.58.13	spark03
spark版本：spark-2.1.0-bin-hadoop2.7
kafka版本：kafka_2.11-2.0.0

Spark Streaming程序

package com.kk.sparkstreaming.kafka

import org.apache.log4j.Level
import org.apache.log4j.Logger
import org.apache.spark.SparkConf
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds
import kafka.serializer.StringDecoder
import org.apache.spark.streaming.kafka.KafkaUtils
import java.sql.DriverManager
import java.sql.PreparedStatement
import java.sql.Connection

object KafkaDirect {

  def main(args: Array[String]): Unit = {

    // 减少日志输出
    Logger.getLogger("org.apache.spark").setLevel(Level.ERROR)
    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)

    val sparkConf = new SparkConf().setAppName("KafkaDirect").setMaster("local[2]")

    val sparkStreaming = new StreamingContext(sparkConf, Seconds(1))

    // 创建topic名称
    val topic = Set("kevin")

    // 制定Kafka的broker地址
    val kafkaParams = Map[String, String]("metadata.broker.list" -> "192.168.58.11:9092")

    // 创建DStream，接受kafka数据irectStream[String, String, StringDecoder,StringDecoder](sparkStreaming, kafkaParams, topic)
    val kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](sparkStreaming, kafkaParams, topic)

    val line = kafkaStream.map(e => {
      new String(e.toString())
    })

    // 获取数据
    val logRDD = kafkaStream.map(_._2)
    // 将数据打印在屏幕
    logRDD.print()

    // 对接受的数据进行分词处理
    val datas = logRDD.map(line => {
      // 1,201.105.101.108,http://mystore.jsp/?productid=1,2017020029,2,1
      val index: Array[String] = line.split(",")
      val ip = index(1);
      (ip, 1)
    })

    // 打印在屏幕
    datas.print()

    // 将数据保存在mysql数据库
    datas.foreachRDD(cs => {
      var conn: Connection = null;
      var ps: PreparedStatement = null;
      try {
        Class.forName("com.mysql.jdbc.Driver").newInstance();
        cs.foreachPartition(f => {
          conn = DriverManager.getConnection("jdbc:mysql://192.168.58.14:3306/storm?useUnicode=true&characterEncoding=utf8", "root", "kevin");
          ps = conn.prepareStatement("insert into result values(?,?)");
          f.foreach(s => {
            ps.setString(1, s._1);
            ps.setInt(2, s._2);
            ps.executeUpdate();
          })
        })
      } catch {
        case t: Throwable => t.printStackTrace() // TODO: handle error
      } finally {
        if (ps != null) {
          ps.close()
        }
        if (conn != null) {
          conn.close();
        }
      }
    })

    sparkStreaming.start()
    sparkStreaming.awaitTermination()
  }
}

pom.xml文件

	UTF-8
	2.2.1
	2.11.1




	
		org.scala-lang
		scala-library
		${scala.version}
	
	
	
		org.scala-lang
		scala-compiler
		${scala.version}
	
	
	
		org.scala-lang
		scala-reflect
		${scala.version}
	
	
	
		org.apache.spark
		spark-core_2.11
		${spark.version}
	
	
	
		org.apache.spark
		spark-streaming_2.11
		${spark.version}
	
	
	
		org.apache.spark
		spark-sql_2.11
		${spark.version}
	
	
	
		org.apache.spark
		spark-streaming-kafka-0-8_2.11
		2.1.1
	

	
		mysql
		mysql-connector-java
		5.1.8
	

测试
1.启动kafka集群，创建消息的生产者
2.将程序提交至集群运行
集群运行出现异常，jar包冲突

Exception in thread "main" java.lang.ClassCastException: kafka.cluster.BrokerEndPoint cannot be cast to kafka.cluster.Broker
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3$$anonfun$apply$6$$anonfun$apply$7.apply(KafkaCluster.scala:90)
	at scala.Option.map(Option.scala:145)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3$$anonfun$apply$6.apply(KafkaCluster.scala:90)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3$$anonfun$apply$6.apply(KafkaCluster.scala:87)
	at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)
	at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)
	at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
	at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
	at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251)
	at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3.apply(KafkaCluster.scala:87)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2$$anonfun$3.apply(KafkaCluster.scala:86)
	at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)
	at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)
	at scala.collection.immutable.Set$Set1.foreach(Set.scala:74)
	at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251)
	at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2.apply(KafkaCluster.scala:86)
	at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$2.apply(KafkaCluster.scala:85)
	at scala.util.Either$RightProjection.flatMap(Either.scala:523)

解决方法
需将kafak的jar包替换成kafka_2.11-0.8.2.1.jar，复制在saprk每个节点的jars目录下