sparkStreaming接收kafka消息，写入数据库

sparkStreaming接收kafka消息，写入数据库

package com.ruijie.spark.stream

import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import com.alibaba.fastjson.JSON
import com.ruijie.bean.DeviceRscUsg
import com.ruijie.dao.DeviceRscUsgDao
import org.apache.spark.streaming.dstream.DStream

/**
  * @ ClassName DevicePortConsumer
  * @ Description TODO
  * @ Author LXW
  * @ Date 2020-06-06 13:57
  **/
object DevicePortConsumer {
  def main(args: Array[String]): Unit = {
    // offset保存路径
    val checkpointPath = "D:\\hadoop\\checkpoint\\kafka-direct"

    val conf = new SparkConf()
      .setAppName("devicePortConsumer")
      .setMaster("local[2]")

    val sc = new SparkContext(conf)
    sc.setLogLevel("WARN")

    val ssc = new StreamingContext(sc, Seconds(5))
    ssc.checkpoint(checkpointPath)

    val bootstrapServers = "172.18.130.210:9092"
    val groupId = "lxw"
    val topicName = "dev-port"
    val maxPoll = 500

    val kafkaParams = Map(
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> bootstrapServers,
      ConsumerConfig.GROUP_ID_CONFIG -> groupId,
      ConsumerConfig.MAX_POLL_RECORDS_CONFIG -> maxPoll.toString,
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer]
    )

    val kafkaTopicDS = KafkaUtils.createDirectStream(ssc, LocationStrategies.PreferConsistent,
      ConsumerStrategies.Subscribe[String, String](Set(topicName), kafkaParams))

    val value: DStream[String] = kafkaTopicDS.map(_.value())
    value.map(devPortMessage => {
      print(devPortMessage)
      val deviceRscUsg: DeviceRscUsg = JSON.parseObject(devPortMessage, classOf[DeviceRscUsg])
      deviceRscUsg.setTs(new java.util.Date(deviceRscUsg.getTimestamp))
      DeviceRscUsgDao.insert(deviceRscUsg)
    }).print()

    ssc.start()
    ssc.awaitTermination()
  }
}