Flink消费Redis Stream数据
前言
对于流处理，感觉flink近乎苛刻的只对kafka友好。当然我对kafka也有天然的好感，但是相对于redis而言，kafka还是稍显复杂了一些。我们的生产环境中没有kafka，只有redis。装一套kafka集群可以吗。由于业务长期的累积，引入一套全新的架构真的是难如登天。所以只能委屈求全，在我们的业务系统中准备使用redis作为flink的数据源。
幸运的是，在redis5中已经有原生支持消息队列的数据存储结构了，即stream。但是现在网上介绍和使用redis stream的并不多。常用的redis客户端redisTemplate、jedis还没有支持,只有Redisson和Lettuce支持了。
所以这先抛砖引玉，如果各位读者有更好的redis source解决方案可以介绍一下，感谢。
Redis配置
为了方便介绍，我这里使用Spring注入的方式定义各个对象，各位完全不必如此定义。
package it.aspirin.demo.config;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.api.sync.RedisCommands;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@EnableAutoConfiguration
@Configuration
public class RedisConfig {
@Value("${redis.database.flink}")
private int flinkDb;
@Value("${spring.redis.host}")
private String host;
@Value("${spring.redis.port}")
private int port;
@Value("${spring.redis.password}")
private String password;
@Value("${spring.redis.timeout}")
private int timeout;
/**
* stream的各种操作命令主要使用RedisCommands对象进行
* @return
*/
@Bean(name = "streamRedisCommands")
public RedisCommands getRedisTemplate(){
RedisURI redisURI = new RedisURI();
redisURI.setHost(host);
redisURI.setPort(port);
redisURI.setDatabase(flinkDb);
RedisClient redisClient = RedisClient.create(redisURI);
StatefulRedisConnection connect = redisClient.connect();
return connect.sync();
}
@Bean
public RedisConnectionFactory redisConnectionFactory() {
return new LettuceConnectionFactory(new RedisStandaloneConfiguration(host, port));
}
@Bean
public RedisTemplate redisTemplate(RedisConnectionFactory redisConnectionFactory) {
RedisTemplate redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(redisConnectionFactory);
// 可以配置对象的转换规则，比如使用json格式对object进行存储。
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.setValueSerializer(new StringRedisSerializer());
return redisTemplate;
}
} 封装redis生产者。flink是消息队列的消费者，因此下面对象，flink中并不会用到。
package it.aspirin.demo.redis;
import io.lettuce.core.api.sync.RedisCommands;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
/**
* 往redis中生产数据
*/
@Component
public class RedisProducer {
private final Logger logger = LoggerFactory.getLogger(RedisProducer.class);
@Resource(name = "streamRedisCommands")
public void setRedisClient(RedisCommands redisSyncCommands) {
RedisProducer.redisSyncCommands = redisSyncCommands;
}
private static RedisCommands redisSyncCommands;
public void send(String streamKey, String... message) {
try {
//第一个参数为stream的key，后面是内容
String recordId = redisSyncCommands.xadd(streamKey, message);
logger.info("send message successful {}", recordId);
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}
}
} 封装消费者。
package it.aspirin.demo.redis;
import io.lettuce.core.Consumer;
import io.lettuce.core.StreamMessage;
import io.lettuce.core.XReadArgs;
import io.lettuce.core.api.sync.RedisCommands;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.List;
/**
* Redis 消费者
*/
@Component
public class RedisConsumer {
private final Logger logger = LoggerFactory.getLogger(RedisConsumer.class);
@Resource(name = "streamRedisCommands")
public void setRedisClient(RedisCommands redisCommands) {
RedisConsumer.redisCommands = redisCommands;
}
private static RedisCommands redisCommands;
/**
* 判断是否存在消费者组
*
* @param groupName 消费者组名称
* @return
*/
public boolean exists(String groupName) {
Long exists = redisCommands.exists(groupName);
return exists.intValue() == 0;
}
// 普通消费 -- 最后一条消息
public void consumer(String consumerGroup, String streamKey) {
List> streamSmsSend = redisCommands.xread(XReadArgs.StreamOffset.from(streamKey, "0"));
for (StreamMessage message : streamSmsSend) {
System.out.println(message);
redisCommands.xack(streamKey, consumerGroup, message.getId());
}
}
public void createGroup(String consumerGroup, String streamKey) {
// 创建分组
redisCommands.xgroupCreate(XReadArgs.StreamOffset.from(streamKey, "0"), consumerGroup);
}
public void consumerGroup(String consumerGroup, String streamKey) {
// 按组消费
List> xReadGroup = redisCommands.xreadgroup(Consumer.from(consumerGroup, "consumer_1"), XReadArgs.StreamOffset.lastConsumed(streamKey));
for (StreamMessage message : xReadGroup) {
System.out.println("ass - " + message);
// 告知 redis，消息已经完成了消费
redisCommands.xack(streamKey, consumerGroup, message.getId());
}
}
/**
* 读取redis中的数据
*
* @param consumerGroup 消费组
* @param streamKey stream对应的key
* @return
*/
public List> getMessage(String consumerGroup, String streamKey) {
// 按组消费
return redisCommands.xreadgroup(Consumer.from(consumerGroup, "consumer_1"), XReadArgs.StreamOffset.lastConsumed(streamKey));
}
} 自定义redis source function
package it.aspirin.demo.flink.source;
import io.lettuce.core.StreamMessage;
import io.lettuce.core.api.sync.RedisCommands;
import it.aspirin.demo.redis.RedisConsumer;
import it.aspirin.demo.utl.AppUtil;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* 读取redis stream中的数据消费
*/
public class RedisSourceFunction extends RichParallelSourceFunction> {
private final Logger logger = LoggerFactory.getLogger(RedisSourceFunction.class);
private final String consumerGroup = "consumer-group-1";
private final String streamKey = "stream1";
private RedisConsumer consumer;
private RedisCommands redisCommands;
/**
* 创建消费者组
* @param parameters
* @throws Exception
*/
@Override
public void open(Configuration parameters) throws Exception {
consumer = AppUtil.context.getBean(RedisConsumer.class);
redisCommands = AppUtil.context.getBean(RedisCommands.class);
//如果消费者组不存在，则创建
if (!consumer.exists(consumerGroup)) {
consumer.createGroup(consumerGroup, streamKey);
}
}
@Override
public void close() throws Exception {
super.close();
}
/**
* 下面是消费并解析redis中的数据，然后将数据发往flink下游算子
* @param sourceContext
* @throws Exception
*/
@Override
public void run(SourceContext> sourceContext) throws Exception {
try{
while (true) {
List> messages = consumer.getMessage(consumerGroup, streamKey);
for (StreamMessage msg : messages) {
Map body = msg.getBody();
Set keySet = body.keySet();
for (String key : keySet) {
sourceContext.collect(new Tuple2<>(key, body.get(key)));
//因为没有找到让redis中数据过期的方法，因此当消费完一条数据以后将redis中的数据删除，这并不是很严谨的方式
redisCommands.xdel(streamKey, msg.getId());
Long xlen = redisCommands.xlen(streamKey);
logger.info("xlen = {}", xlen);
}
}
}
}catch (Exception e){
String message = e.getMessage();
if (message.contains("Connection reset by peer")){
logger.error("redis maybe shutdown "+ e);
}
}
}
@Override
public void cancel() {
}
}
注意事项
-
只能说上面方式可以消费redis队列中的数据，但是不能保证性能很好，如有可以优化的地方，欢迎指正
-
我们没有有找到如何使redis stream中的数据过期，如果数据是长期存储的，需要确定redis是否吃得消。我的解决办法是消费一条数据，接着将该数据删除，这并不是一种很好的处理方式。
-
还有很多跟stream操作相关的api，如有需要可以自行学习，redisCommands中以
x开头的命令都是与stream相关的命令。