package com.cn.util
import java.util.Properties
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
import org.codehaus.jettison.json.JSONObject
import scala.util.Random
/**
* 编写一个提交数据到kafka集群的producer
* 模拟场景：
* 统计一些用户实时步行的总步数，每隔5s统计一次，包括某个用户新统计时的时间、所在地点、新增步数；
*/
object KafkaEventProducer {
//用户
private val users = Array(
"zhangSan", "liSi",
"wangWu", "xiaoQiang",
"zhangFei", "liuBei",
"guanYu", "maChao",
"caoCao", "guanYu"
)
private var pointer = -1
//随机获得用户
def getUser(): String = {
pointer = (pointer + 1) % users.length
users(pointer)
}
//获取新增步数
val random = new Random()
def getNewStepNum(): Int = {
random.nextInt(users.length)
}
//获取统计时间
def getTime(): Long = {
System.currentTimeMillis()
}
//获取行走地点
val walkPlace = Array(
"操场南门", "操场东门", "操场北门", "操场西门", "操场东南门", "操场西北门", "操场西南门", "操场东南北门"
)
def getWalkPlace(): String = {
walkPlace(random.nextInt(walkPlace.length))
}
def main(args: Array[String]): Unit = {
val topic = "topic_walkCount"
val brokers = "master:6667,slaves1:6667,slaves2:6667"
//设置属性,配置
val props = new Properties()
props.setProperty("bootstrap.servers", brokers)
props.setProperty("metadata.broker.list", brokers)
props.setProperty("key.serializer", "org.apache.kafka.common.serialization.StringSerializer")
props.setProperty("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")
//生成producer对象
val producer = new KafkaProducer[String, String](props)
//传输数据
while (true) {
val event = new JSONObject()
event.put("user", getUser())
.put("count_time", getTime())
.put("walk_place", getWalkPlace())
.put("new_walkNum", getNewStepNum())
println(event.toString())
//发送数据
producer.send(new ProducerRecord[String, String](topic, event.toString))
Thread.sleep(5000)
}
}
}
3、redis工具类
package com.cn.util
import redis.clients.jedis.JedisPool
object RedisUtils {
private val host = "master"
private val port = 6379
//private val poolConfig = new GenericObjectPoolConfig()
lazy val pool = new JedisPool(host, port)
//关闭
lazy val hooks = new Thread() {
override def run(): Unit = {
println("Execute hook thread: " + this)
pool.destroy()
}
}
}
4、业务处理代码
package com.cn.sparkStreaming
import com.cn.util.RedisUtils
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010._
import org.codehaus.jettison.json.JSONObject
/**
* 统计一些用户实时步行的总步数，每隔5s统计一次，包括某个用户新统计时的时间、所在地点、新增步数；
* 将每个用户以及实时更新的步数总和保存到redis数据库中；
*/
object kafka2sparkStreaming2redis {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("kafka2sparkStreaming2redis")
.setMaster("local[1]")
//.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
//设置流数据每批的时间间隔为2s
val ssc = new StreamingContext(conf, Seconds(2))
//控制日志输出级别
ssc.sparkContext.setLogLevel("WARN") //WARN,INFO,DEBUG
ssc.checkpoint("checkpoint")
val topic = "topic_walkCount"
val groupId = "t03"
val kafkaParams = Map[String, Object](
"bootstrap.servers" -> "master:6667,slaves1:6667,slaves2:6667",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> groupId,
"auto.offset.reset" -> "earliest", // 初次启动从最开始的位置开始消费
"enable.auto.commit" -> (false: java.lang.Boolean) // 自动提交设置为 false
)
val topics = Array(topic)
val stream: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](
ssc,
LocationStrategies.PreferConsistent, //均匀分发到executor
ConsumerStrategies.Subscribe[String, String](topics, kafkaParams)
)
val dbIndex = 3;
stream.foreachRDD(rdd => {
// 获取每一个分区的消费的偏移量
val offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
rdd.foreachPartition(partitions => {
partitions.foreach(records => {
val record = new JSONObject(records.value())
val user = record.getString("user")
val countTime = record.getLong("count_time")
val walkPlace = record.getString("walk_place")
val newWalkNum = record.getInt("new_walkNum")
//获取redis对象
val jedis = RedisUtils.pool.getResource
//redis密码
jedis.auth("123456")
//选择数据库
jedis.select(dbIndex)
val count = jedis.hincrBy("user_walknum", user, newWalkNum)
println(count)
RedisUtils.pool.returnResource(jedis)
})
})
// 手动提交偏移量
stream.asInstanceOf[CanCommitOffsets].commitAsync(offsetRanges)
})
ssc.start()
ssc.awaitTermination()
}
}
三、运行及结果展示
1、模拟数据展示
log4j:WARN No appenders could be found for logger (org.apache.kafka.clients.producer.ProducerConfig).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
{"user":"zhangSan","count_time":1582689943955,"walk_place":"操场北门","new_walkNum":5}
{"user":"liSi","count_time":1582689956236,"walk_place":"操场东门","new_walkNum":0}
{"user":"wangWu","count_time":1582689961236,"walk_place":"操场东南北门","new_walkNum":2}
{"user":"xiaoQiang","count_time":1582689966239,"walk_place":"操场东门","new_walkNum":6}
{"user":"zhangFei","count_time":1582689971240,"walk_place":"操场东门","new_walkNum":8}
{"user":"liuBei","count_time":1582689976240,"walk_place":"操场西南门","new_walkNum":5}
{"user":"guanYu","count_time":1582689981240,"walk_place":"操场东南门","new_walkNum":9}
{"user":"maChao","count_time":1582689986240,"walk_place":"操场北门","new_walkNum":6}
{"user":"caoCao","count_time":1582689991245,"walk_place":"操场东南北门","new_walkNum":2}
{"user":"guanYu","count_time":1582689996245,"walk_place":"操场西门","new_walkNum":0}
{"user":"zhangSan","count_time":1582690001246,"walk_place":"操场东门","new_walkNum":3}
{"user":"liSi","count_time":1582690006247,"walk_place":"操场北门","new_walkNum":2}
