Redis消息队列
本文主要介绍三种实现方式文末有demo链接
-
redis原生消息队列实现 redis版本必须大于5.0
-
redis rightPop实现消息队列
-
reids ZSet实现延迟消息队列
1.redis stream
Redis Stream 是 Redis 5.0 版本新增加的数据结构。
Redis Stream 主要用于消息队列（MQ，Message Queue），Redis 本身是有一个 Redis 发布订阅 (pub/sub) 来实现消息队列的功能，但它有个缺点就是消息无法持久化，如果出现网络断开、Redis 宕机等，消息就会被丢弃。
配置消费者监听
package example.config;
import example.annotation.MeiceRedisStreamListener;
import example.bean.MeiceUser;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.connection.stream.*;
import org.springframework.data.redis.core.StreamOperations;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.serializer.StringRedisSerializer;
import org.springframework.data.redis.stream.StreamListener;
import org.springframework.data.redis.stream.StreamMessageListenerContainer;
import org.springframework.data.redis.stream.Subscription;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.lang.reflect.Method;
import java.time.Duration;
import java.util.Map;
import java.util.Vector;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ForkJoinPool;
/**
* 利用redis原生消息队列实现
* Copyright © 2021 meicet. All rights reserved.
* @author zyx
* @date 2021-06-21 09:49:11
*/
@Slf4j
@Configuration
@Component
public class RedisConsumerConfig implements DisposableBean {
@Resource
private ApplicationContext context;
@Resource
private StringRedisTemplate stringRedisTemplate;
private Vector>> containerList = new Vector<>();
@Resource
ForkJoinPool forkJoinPool;
@Resource
RedisConnectionFactory factory;
@Bean(name = "forkJoinPool")
public ExecutorService forkJoinPool() {
return new ForkJoinPool();
}
@PostConstruct
public void initRedisStream() throws Exception {
Map beansWithAnnotation = context.getBeansWithAnnotation(MeiceRedisStreamListener.class);
if (beansWithAnnotation.size() == 0) {
return;
}
for (Object item : beansWithAnnotation.values()) {
if (!(item instanceof StreamListener)) {
continue;
}
Method method = item.getClass().getDeclaredMethod("onMessage", Record.class);
MeiceRedisStreamListener annotation = method.getAnnotation(MeiceRedisStreamListener.class);
if (annotation == null) {
continue;
}
creasteSubscription(factory, (StreamListener) item, annotation.streamKey(), annotation.consumerGroup(), annotation.consumerName());
}
}
private void creatGroup(String key, String group) {
StreamOperations streamOperations = this.stringRedisTemplate.opsForStream();
String groupName = streamOperations.createGroup(key, group);
log.info("creatGroup:{}", groupName);
}
private Subscription creasteSubscription(RedisConnectionFactory factory, StreamListener streamListener, String streamKey, String group, String consumerName) {
StreamOperations streamOperations = this.stringRedisTemplate.opsForStream();
if (stringRedisTemplate.hasKey(streamKey)) {
StreamInfo.XInfoGroups groups = streamOperations.groups(streamKey);
if (groups.isEmpty()) {
creatGroup(streamKey, group);
} else {
groups.stream().forEach(g -> {
log.info("XInfoGroups:{}", g);
StreamInfo.XInfoConsumers consumers = streamOperations.consumers(streamKey, g.groupName());
log.info("XInfoConsumers:{}", consumers);
});
}
} else {
creatGroup(streamKey, group);
}
StreamMessageListenerContainer.StreamMessageListenerContainerOptions> options =
StreamMessageListenerContainer.StreamMessageListenerContainerOptions.builder()
.batchSize(10)
.serializer(new StringRedisSerializer())
.executor(forkJoinPool)
.pollTimeout(Duration.ZERO)
.targetType(MeiceUser.class)
.build();
StreamMessageListenerContainer> listenerContainer = StreamMessageListenerContainer.create(factory, options);
StreamOffset streamOffset = StreamOffset.create(streamKey, ReadOffset.lastConsumed());
Consumer consumer = Consumer.from(group, consumerName);
Subscription subscription = listenerContainer.receive(consumer, streamOffset, streamListener);
listenerContainer.start();
this.containerList.add(listenerContainer);
return subscription;
}
@Override
public void destroy() {
this.containerList.forEach(StreamMessageListenerContainer::stop);
}
}
消费者代码
package example.listener;
import cn.hutool.core.date.DateUtil;
import example.annotation.MeiceRedisStreamListener;
import example.bean.MeiceUser;
import example.constant.RedisConstant;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.connection.stream.ObjectRecord;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.stream.StreamListener;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
@MeiceRedisStreamListener
@Component
@Slf4j
public class UserListenerMessage implements StreamListener> {
@Resource
private StringRedisTemplate stringRedisTemplate;
@MeiceRedisStreamListener(streamKey = RedisConstant.STREAM_KEY, consumerGroup = RedisConstant.STREAM_GROUP, consumerName = RedisConstant.CONSUMER_NAME)
@Override
public void onMessage(ObjectRecord message) {
System.err.println("stream:" + DateUtil.current());
stringRedisTemplate.opsForStream().acknowledge(RedisConstant.STREAM_GROUP, message);
MeiceUser value = message.getValue();
System.err.println(value);
}
}
生产者代码
package example.controller;
import cn.hutool.core.date.DateUtil;
import cn.hutool.json.JSONObject;
import example.bean.MeiceUser;
import example.constant.RedisConstant;
import org.springframework.data.redis.connection.stream.ObjectRecord;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
@RestController
public class MsgController {
@Resource
private StringRedisTemplate stringRedisTemplate;
@GetMapping(value = "/{age}")
public String setKey(@PathVariable Integer age) {
MeiceUser user = new MeiceUser();
user.setAge(age);
user.setName("张三");
user.setDate(DateUtil.current());
ObjectRecord stringUserObjectRecord = ObjectRecord.create(RedisConstant.STREAM_KEY, user);
RecordId recordId = stringRedisTemplate.opsForStream().add(stringUserObjectRecord);
stringRedisTemplate.opsForList().leftPush(RedisConstant.QUEUE_MSG, new JSONObject(user).toString());
return new JSONObject(recordId).toString();
}
}
2.redis rightPop
利用redis lpush rightpop实现消息的发送与接受
package example.config;
import example.annotation.MeiceListener;
import example.annotation.RedisListener;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.context.ApplicationContext;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.lang.reflect.Method;
import java.util.Map;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.TimeUnit;
/**
* 利用redis rightPop实现消息队列
* Copyright © 2021 meicet. All rights reserved.
* @author zyx
* @date 2021-06-21 09:43:37
*/
@Slf4j
@Component
public class RedisExecutors implements DisposableBean {
@Resource
private ApplicationContext context;
@Resource
private StringRedisTemplate stringRedisTemplate;
private boolean isClose = false;
@Resource
ForkJoinPool forkJoinPool;
@SneakyThrows
@PostConstruct
public void init() {
Map beanMap = context.getBeansOfType(MeiceListener.class);
if (beanMap.size() == 0) {
return;
}
for (MeiceListener item : beanMap.values()) {
Method method = item.getClass().getDeclaredMethod("onMessage", Object.class);
RedisListener listener = method.getAnnotation(RedisListener.class);
if (listener == null) {
continue;
}
String queue = listener.queue();
log.info("消息队列名称:{}", queue);
forkJoinPool.execute(() -> {
long timeout = listener.timeout();
while (!isClose) {
try {
Object message = stringRedisTemplate.opsForList().rightPop(queue, timeout, TimeUnit.SECONDS);
if (message == null) {
continue;
}
item.onMessage(message);
} catch (Exception e) {
log.error("队列{}读取消息失败:{}", queue, e);
try {
TimeUnit.MILLISECONDS.sleep(1000);
} catch (InterruptedException e1) {
log.error("sleep失败:", e1);
}
}
}
});
}
}
@Override
public void destroy() {
isClose = true;
forkJoinPool.shutdown();
}
} 3.延时队列
利用redis ZSet结构实现延时消息队列
package example.config;
import example.annotation.MeiceListener;
import example.annotation.RedisDelayQueueListener;
import example.utils.RedisLock;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.context.ApplicationContext;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ZSetOperations;
import org.springframework.stereotype.Component;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import java.lang.reflect.Method;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* 延迟队列执行器
* Copyright © 2020 meicet. All rights reserved.
*
* @author zyx
* @date 2020-10-21 10:58:29
*/
@Slf4j
@Component
public class RedisDelayQueueExecutors implements InitializingBean {
@Resource
RedisLock redisLock;
@Resource
private ApplicationContext context;
@Resource
private RedisTemplate redisTemplate;
private AtomicBoolean isClose = new AtomicBoolean(false);
private ExecutorService executorService;
@SneakyThrows
@Override
public void afterPropertiesSet() {
Map beanMap = context.getBeansOfType(MeiceListener.class);
if (beanMap.size() == 0) {
return;
}
executorService = Executors.newFixedThreadPool(beanMap.size());
for (MeiceListener consumer : beanMap.values()) {
Method method = consumer.getClass().getDeclaredMethod("onMessage", Object.class);
RedisDelayQueueListener listener = method.getAnnotation(RedisDelayQueueListener.class);
if (listener == null) {
continue;
}
String queue = listener.queue();
log.info("延迟消息队列名称:{}", queue);
executorService.execute(() -> {
while (!isClose.get()) {
try {
Set> typedTuples = redisTemplate.opsForZSet().rangeWithScores(queue, 0, 0);
if (typedTuples == null || typedTuples.isEmpty()) {
TimeUnit.MILLISECONDS.sleep(500);
continue;
}
Iterator> iterator = typedTuples.iterator();
while (iterator.hasNext()) {
ZSetOperations.TypedTuple
完整代码redis-stream-demo: 利用redis rightPop 和 redis stream 实现消息队列 (gitee.com)
参考:Stream消息队列在SpringBoot中的实践与踩坑 | Lolico's Blog