sparkStreaming的offset保存在redis中
实现功能：
从kafka读取某一主题，消费者组的偏移量
基于读出的offset，创建kafka读取流程
把各个分区的偏移量 保存到redis。
-
-
import Kafka010.Utils.{MyKafkaUtils, RedisUtilsDemo}
-
import org.apache.kafka.clients.consumer.ConsumerRecord
-
import org.apache.kafka.common.TopicPartition
-
import org.apache.spark.SparkConf
-
import org.apache.spark.streaming.dstream.InputDStream
-
import org.apache.spark.streaming.kafka010._
-
import org.apache.spark.streaming.{Seconds, StreamingContext}
-
-
object Test{
-
def main(args:
Array[
String]): Unit = {
-
//创建spark环境
-
val conf =
new SparkConf()
-
.setMaster(
"local[*]")
-
.setAppName(s
"${this.getClass.getCanonicalName}")
-
//创建sparkStreamingContext
-
val ssc =
new StreamingContext(conf, Seconds(
2))
-
//创建kafka相关参数
-
val groupId =
"SparkKafka010"
-
val topics = List(
"datacollection")
-
//获取kafka参数，这里是自定义的MyKafkaUtils类，后面给出类的具体内容
-
val kafkaParams = MyKafkaUtils.getKafkaConsumerParams(groupId,
"false")
-
-
// 从redis读取offset
-
val offsets:
Map[TopicPartition, Long] = RedisUtilsDemo.getOffsetFromRedis(
"datacollection", groupId)
-
//获取kafka数据
-
val ds: InputDStream[ConsumerRecord[
String,
String]] = KafkaUtils.createDirectStream[
String,
String](ssc,
-
LocationStrategies.PreferConsistent,
-
ConsumerStrategies.Subscribe[
String,
String](topics, kafkaParams,offsets)
-
)
-
-
ds.foreachRDD(
rdd => {
-
val ranges:
Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
-
-
// 代表对数据进行处理
-
if (! rdd.isEmpty())
-
println(rdd.count)
-
-
// 代表对offset进行处理
-
ranges.foreach(
offset =>
-
println(s
"${offset.partition}, ${offset.fromOffset}, ${offset.untilOffset}")
-
)
-
//保存offset到redis
-
RedisUtilsDemo.saveOffsetToRedis(ranges, groupId)
-
})
-
-
ssc.start()
-
ssc.awaitTermination()
-
}
-
}
MyKafkaUtils工具类
-
import org.apache.kafka.clients.consumer.ConsumerConfig
-
import org.apache.kafka.common.serialization.StringDeserializer
-
-
object MyKafkaUtils {
-
def getKafkaConsumerParams(groupId:
String =
"SparkStreaming010",
autoCommit:
String=
"true"):
Map[
String,
String] = {
-
val kafkaParams =
Map[
String,
String](
-
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG ->
"mini1:9092,mini2:9092",
-
ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer].getName,
-
ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer].getName,
-
ConsumerConfig.GROUP_ID_CONFIG -> groupId,
-
ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG -> autoCommit)
-
kafkaParams
-
}
-
-
def main(args:
Array[
String]): Unit = {
-
-
}
-
}
RedisUtilDemo工具类
-
-
import cn.bigdata.antispider.common.util.jedis.{JedisConUtil, JedisConnectionUtil}
-
import org.apache.kafka.common.TopicPartition
-
import org.apache.spark.streaming.kafka010.OffsetRange
-
-
-
object RedisUtilsDemo {
-
//获取jedis连接
-
private
val jedis = JedisConUtil.getJedisClient()
-
// 读取offset
-
// key(kafka:topic:groupid) value(parition:offset;parition:offset)
-
// 没有考虑 key 不存在的情况，为了方便测试可以先提前set一下
-
def getOffsetFromRedis(topic: String, groupid: String): Map[TopicPartition,
Long] = {
-
val key = s
"kafka:$topic:$groupid"
-
val offsetStr: String = jedis.
get(key)
-
-
offsetStr.split(
";").map(str => {
-
val fileds = str.split(
":")
-
val parition:
Int = fileds.head.toInt
-
val offset:
Long = fileds.last.toLong
-
(new TopicPartition(topic, parition) -> offset)
-
}).toMap
-
-
}
-
-
// 可以给定多个topic；返回多个topic对应的对应的offset
-
def getOffsetFromRedis2(topics: Iterator[String], groupid: String): Iterator[Option[Map[TopicPartition,
Long]]] = {
-
topics.map { topic =>
-
val key = s
"kafka:$topic:$groupid"
-
val offsetStr: String = jedis.
get(key)
-
-
if (offsetStr !=
null && offsetStr.trim.size>
0 ) {
-
val offsets = offsetStr.split(
";").map { str =>
-
val fileds = str.split(
":")
-
val parition:
Int = fileds.head.toInt
-
val offset:
Long = fileds.last.toLong
-
(new TopicPartition(topic, parition) -> offset)
-
}.toMap
-
Some(offsets)
-
}
-
else None
-
}
-
}
-
-
def getOffsetFromRedis1(topic: String, groupid: String): Option[Map[TopicPartition,
Long]] = {
-
val key = s
"kafka:$topic:$groupid"
-
val offsetStr: String = jedis.
get(key)
-
-
if (offsetStr !=
null && offsetStr.trim.size>
0 ) {
-
val offsets = offsetStr.split(
";").map(str => {
-
val fileds = str.split(
":")
-
val parition:
Int = fileds.head.toInt
-
val offset:
Long = fileds.last.toLong
-
(new TopicPartition(topic, parition) -> offset)
-
}).toMap
-
Some(offsets)
-
}
-
else
-
None
-
}
-
-
// 保存offset
-
// key(kafka:topic:groupid) value(parition:offset;parition:offset)
-
def saveOffsetToRedis(ranges: Array[OffsetRange], groupId: String):
Unit = {
-
ranges.map(offsets => (offsets.topic, (offsets.partition, offsets.untilOffset)))
-
.groupBy(_._1)
-
.foreach{case ((topic, buffer)) =>
-
val key = s
"kafka:$topic:$groupId"
-
val value = buffer.map{case (_, (partition, untilOffset)) => s
"$partition:$untilOffset"}.mkString(
";")
-
jedis.
set(key, value)
-
}
-
}
-
-
-
-
}