【redis】stream消息队列

目录

        • 截图
        • 一、代码示例
          • 1.1 pom.xml依赖
          • 1.2 application.xml配置
          • 1.3 启动类
          • 1.4 配置类
          • 1.5 消息实体
          • 1.6 自定义错误
          • 1.7 自定义注解
          • 1.8 服务类
          • 1.9 监听类
          • 1.10 controller

截图

【redis】stream消息队列_第1张图片
【redis】stream消息队列_第2张图片

一、代码示例

1.1 pom.xml依赖


    
        springboot-learning
        com.learning
        1.0-SNAPSHOT
    
    4.0.0

    springboot-mq-redis

    
        8
        8
    

    
        
            org.springframework.boot
            spring-boot-starter-web
        

        
            org.springframework.boot
            spring-boot-starter-test
        
        
            org.springframework.data
            spring-data-redis
            2.5.11
        
        
            org.redisson
            redisson-spring-data-26
            3.17.1
        
        
        
            com.alibaba
            fastjson
            1.1.15
        
        
            org.apache.commons
            commons-lang3
        
    

    
        
            
                org.springframework.boot
                spring-boot-maven-plugin
            
        
    

1.2 application.xml配置
spring:
  redis:
    host: 127.0.0.1
    port: 6379
    password:
    database: 0
    ssl: false
redis:
  model: 1
1.3 启动类
package com.learning.mq.redis;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

/**
 * @Description 启动类
 */

@SpringBootApplication
public class Application {
    public static void main(String[] args) {
        SpringApplication.run(Application.class, args);
    }
}
1.4 配置类
package com.learning.mq.redis.config;

import lombok.Data;

@Data
public class RedisClusterProperties {

    /**
     * 集群状态扫描间隔时间,单位是毫秒
     */
    private int scanInterval=5000;

    /**
     * 集群节点
     */
    private String nodes;

    /**
     * 默认值: SLAVE(只在从服务节点里读取)设置读取操作选择节点的模式。 可用值为: SLAVE - 只在从服务节点里读取。
     * MASTER - 只在主服务节点里读取。 MASTER_SLAVE - 在主从服务节点里都可以读取
     */
    private String readMode;
    /**
     * (从节点连接池大小) 默认值:64
     */
    private int slaveConnectionPoolSize;
    /**
     * 主节点连接池大小)默认值:64
     */
    private int masterConnectionPoolSize;

    /**
     * (命令失败重试次数) 默认值:3
     */
    private int retryAttempts;

    /**
     *命令重试发送时间间隔,单位:毫秒 默认值:1500
     */
    private int retryInterval;

    /**
     * 执行失败最大次数默认值:3
     */
    private int failedAttempts;

}
package com.learning.mq.redis.config;

import lombok.Data;

@Data
public class RedisPoolProperties {

    private int maxIdle;

    private int minIdle;

    private int maxActive;

    private int maxWait;

    private int connTimeout;

    private int soTimeout = 30000;
    /**
     * 池大小
     */
    private  int size;

}

package com.learning.mq.redis.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

@Component
@ConfigurationProperties(prefix = "spring.redis")
@Data
public class RedisProperties {

    private int database;

    /**
     * 等待节点回复命令的时间。该时间从命令发送成功时开始计时
     */
    private int timeout;

    private String password;

    /**
     * 池配置
     */
    private RedisPoolProperties pool;

    /**
     * 集群 信息配置
     */
    private RedisClusterProperties cluster;

    /**
     * 哨兵配置
     */
    private RedisSentinelProperties sentinel;

    private  String host;
    private  String port;

    public String getAddress() {
        return this.getHost()+":"+this.getPort();
    }

}

package com.learning.mq.redis.config;

import lombok.Data;

@Data
public class RedisSentinelProperties {

    /**
     * 哨兵master 名称
     */
    private String master;

    /**
     * 哨兵节点
     */
    private String nodes;

    /**
     * 哨兵配置
     */
    private boolean masterOnlyWrite;

    /**
     *
     */
    private int failMax;

}

package com.learning.mq.redis.config;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.redisson.Redisson;
import org.redisson.api.RedissonClient;
import org.redisson.config.*;
import org.redisson.spring.data.connection.RedissonConnectionFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

@Configuration
@EnableConfigurationProperties(RedisProperties.class)
@Slf4j
public class RedissonConfig {
    @Autowired
    private RedisProperties redisProperties;
    @Value("${redis.model}")
    private String mode;
    @Value("${spring.redis.database:0}")
    private String database;

    /**
     * 单机模式 redisson 客户端
     */
    @Bean
    @ConditionalOnProperty(name = "redis.model", havingValue = "1")
    public RedissonClient redissonSingle() {
        log.info("redis.model=1, 为单机模式");
        Config config = new Config();
        String node = redisProperties.getAddress();
        node = node.startsWith("redis://") ? node : "redis://" + node;
        SingleServerConfig serverConfig = config.useSingleServer()
                .setAddress(node)
                .setDatabase(redisProperties.getDatabase());
//                .setTimeout(redisProperties.getPool().getConnTimeout());
//                .setConnectionPoolSize(redisProperties.getPool().getSize())
//                .setConnectionMinimumIdleSize(redisProperties.getPool().getMinIdle());
        if (StringUtils.isNotBlank(redisProperties.getPassword())) {
            serverConfig.setPassword(redisProperties.getPassword());
        }
        return Redisson.create(config);
    }

    /**
     * 哨兵模式 redisson 客户端
     *
     * @return
     */

    @Primary
    @Bean
    @ConditionalOnProperty(name = "redis.model", havingValue = "2")
    public RedissonClient redissonSentinel() {
        log.info("redis.model=2, 为哨兵模式");
        Config config = new Config();
        String[] nodes = redisProperties.getSentinel().getNodes().split(",");
        List newNodes = new ArrayList(nodes.length);
        Arrays.stream(nodes).forEach((index) -> newNodes.add(index.startsWith("redis://") ? index : "redis://" + index));
        SentinelServersConfig serverConfig = config.useSentinelServers()
                .addSentinelAddress(newNodes.toArray(new String[0]))
                .setDatabase(redisProperties.getDatabase())
                .setMasterName(redisProperties.getSentinel().getMaster())
                .setReadMode(ReadMode.MASTER)//TODO 如果实现读写分离时,需要配置这里
                .setTimeout(redisProperties.getTimeout());
//                .setMasterConnectionPoolSize(redisProperties.getPool().getSize())
//                .setSlaveConnectionPoolSize(redisProperties.getPool().getSize());
        if (StringUtils.isNotBlank(redisProperties.getPassword())) {
            serverConfig.setPassword(redisProperties.getPassword());
        }
        return Redisson.create(config);
    }

    @Primary
    @Bean("redissonConnectionFactory")
    public RedissonConnectionFactory redissonConnectionFactory() {
        RedissonClient redisson;
        if ("1".equals(mode)) {
            redisson = redissonSingle();
        } else if ("2".equals(mode)) {
            redisson = redissonSentinel();
        } else {
            mode = "0";
            redisson = redissonCluster();
        }
        return new RedissonConnectionFactory(redisson);
    }

    /**
     * 集群模式的 redisson 客户端
     *
     * @return
     */
    @Bean
    @ConditionalOnProperty(name = "redis.model", havingValue = "3")
    RedissonClient redissonCluster() {
        log.info("redis.model=3, 为集群模式");
        Config config = new Config();
        String[] nodes = redisProperties.getCluster().getNodes().split(",");
        List newNodes = new ArrayList(nodes.length);
        Arrays.stream(nodes).forEach((index) -> newNodes.add(index.startsWith("redis://") ? index : "redis://" + index));

        ClusterServersConfig serverConfig = config.useClusterServers()
                .addNodeAddress(newNodes.toArray(new String[0]))
                .setScanInterval(redisProperties.getCluster().getScanInterval())
                .setReadMode(ReadMode.MASTER)
//                .setIdleConnectionTimeout(redisProperties.getPool().getSoTimeout())
//                .setConnectTimeout( redisProperties.getPool().getConnTimeout())
//                .setRetryAttempts( redisProperties.getCluster().getRetryAttempts())
//                .setRetryInterval( redisProperties.getCluster().getRetryInterval())
//                .setMasterConnectionPoolSize(redisProperties.getCluster().getMasterConnectionPoolSize())
//                .setSlaveConnectionPoolSize(redisProperties.getCluster().getSlaveConnectionPoolSize())
                .setPingConnectionInterval(1000)
                .setTimeout(redisProperties.getTimeout());
        if (StringUtils.isNotBlank(redisProperties.getPassword())) {
            serverConfig.setPassword(redisProperties.getPassword());
        }
        return Redisson.create(config);
    }

    @Bean
    public RedisTemplate redisTemplate(RedisConnectionFactory redisConnectionFactory) {
        RedisTemplate redisTemplate = new RedisTemplate<>();
        redisTemplate.setConnectionFactory(redisConnectionFactory);
        // 使用Jackson2JsonRedisSerialize 替换默认序列化
        Jackson2JsonRedisSerializer jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer(Object.class);

        ObjectMapper objectMapper = new ObjectMapper();
        objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
        objectMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL);

        jackson2JsonRedisSerializer.setObjectMapper(objectMapper);

        // 设置value的序列化规则和 key的序列化规则
        redisTemplate.setValueSerializer(jackson2JsonRedisSerializer);
        redisTemplate.setKeySerializer(new StringRedisSerializer());

        redisTemplate.setHashKeySerializer(new StringRedisSerializer());
        redisTemplate.setHashValueSerializer(jackson2JsonRedisSerializer);

        redisTemplate.setDefaultSerializer(jackson2JsonRedisSerializer);
        redisTemplate.setEnableDefaultSerializer(true);
        redisTemplate.afterPropertiesSet();
        return redisTemplate;
    }

    @Bean
    public StringRedisTemplate stringRedisTemplate(RedisConnectionFactory redisConnectionFactory) {
        StringRedisTemplate template = new StringRedisTemplate();
        template.setConnectionFactory(redisConnectionFactory);
        return template;
    }
}

1.5 消息实体
package com.learning.mq.redis.dto;

import lombok.Data;

/**
 * @Description 消息实体
 */
@Data
public class MessageDTO {
    private String content;
}
1.6 自定义错误
package com.learning.mq.redis.handler;

import lombok.extern.slf4j.Slf4j;
import org.springframework.util.ErrorHandler;

/**
 * @description
 */
@Slf4j
public class CustomerErrorHandler implements ErrorHandler {
    @Override
    public void handleError(Throwable e) {
        //TODO 消费异常时处理逻辑
        log.error("发生了异常", e);
    }
}

1.7 自定义注解
package com.learning.mq.redis.annotation;

import java.lang.annotation.*;

/**
 * 消息队列自定义注解
 */
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface RedisStreamMessageQueueListener {
    /**
     * 消息队列的 key
     * @return
     */
    String value();

    /**
     * 消费者组
     * @return
     */
    String group() default "";

    /**
     * 消息反序列化类型
     * @return
     */
    Class type();

    /**
     * 是否自动 acknowledge
     * @return
     */
    boolean autoAck() default true;

    /**
     * 每次拉取的消息数,线程池每秒拉取一次
     * @return
     */
    int batchSize() default 5;

    /**
     * 启用死信队列
     * @return
     */
    boolean enableDeadLetter() default false;

    /**
     * 投递次数阈值
     * 超过阈值后,消息将被添加到死信队列中
     * @return
     */
    int maxDeliverTimes() default 3;

    /**
     * 消息未Ack时间阈值(单位:秒)
     * 超过阈值的消息将被重新投递到其他消费者(仅当消费者组存在多个消费者情况)
     * @return
     */
    int timeout() default 300;

    /**
     * 死信队列的key,当enableDeadLetter=true时必填
     * @return
     */
    String deadLetterKey() default "";

    /**
     * 死信队列最大长度
     * @return
     */
    int deadLetterMaxSize() default 1000;

}

1.8 服务类
package com.learning.mq.redis.service;

import com.alibaba.fastjson.JSON;
import com.learning.mq.redis.annotation.RedisStreamMessageQueueListener;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.boot.context.event.ApplicationStartedEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.dao.DataAccessException;
import org.springframework.data.domain.Range;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.RedisStreamCommands;
import org.springframework.data.redis.connection.RedisZSetCommands;
import org.springframework.data.redis.connection.stream.*;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.stream.StreamListener;

import java.time.Duration;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

@Slf4j
public class ListenerAnnotation implements ApplicationListener, InitializingBean {
    protected Logger streamMqLog = LoggerFactory.getLogger(ListenerAnnotation.class);

    private static final String DISTUIBUTE_HANDLE_PENDING_DATA_LOCK = "distuibute_handle_pending_data_lock";

    private static List ans = new ArrayList<>();
    // 记录违规的死信队列keys(违规:死信队列又配置了死信队列)
    private List illegalDeadLetterKeys = new ArrayList<>();

    private volatile static ListenerAnnotation listenerAnnotation;

    private final RedisStreamMessageQueueStartService redisStreamMessageQueueStartService;
    private RedisTemplate redisTemplate;
    private final String applicationName;
    private final RedissonClient redissonClient;

    // 当前服务ip及端口号,用于动态消费者组的游标继承
    private final String serviceIP;
    private final String servicePort;

    private ListenerAnnotation(RedisStreamMessageQueueStartService redisStreamMessageQueueStartService, RedisTemplate redisTemplate, String applicationName, RedissonClient redissonClient, String serviceIP, String servicePort) {
        this.redisStreamMessageQueueStartService = redisStreamMessageQueueStartService;
        this.redisTemplate = redisTemplate;
        this.applicationName = applicationName;
        this.redissonClient = redissonClient;
        this.serviceIP = serviceIP;
        this.servicePort = servicePort;
    }

    /**
     * 单例
     *
     * @param redisStreamMessageQueueStartService
     * @return
     */
    public static ListenerAnnotation getListenerAnnotation(RedisStreamMessageQueueStartService redisStreamMessageQueueStartService, RedisTemplate redisTemplate, String group, RedissonClient redissonClient, String serviceIP, String servicePort) {
        if (listenerAnnotation == null) {
            synchronized (ListenerAnnotation.class) {
                if (listenerAnnotation == null) {
                    listenerAnnotation = new ListenerAnnotation(redisStreamMessageQueueStartService, redisTemplate, group, redissonClient, serviceIP, servicePort);
                }
            }
        }
        return listenerAnnotation;
    }

    /**
     * 循环处理keys中的pending数据
     */
    private void handlePendingData() {
        for (RedisStreamMessageQueueListener redisStreamMessageQueueListener : ans) {
            String key = redisStreamMessageQueueListener.value();
            String groupName = redisStreamMessageQueueListener.group();
            StreamInfo.XInfoGroups groups = redisTemplate.opsForStream().groups(key);
            if (groups.groupCount() > 0) {
                Iterator groupIterator = groups.stream().iterator();
                while (groupIterator.hasNext()) {
                    StreamInfo.XInfoGroup group = groupIterator.next();
                    if (StringUtils.equals(group.groupName(), groupName)) {
                        StreamInfo.XInfoConsumers consumers = redisTemplate.opsForStream().consumers(key, groupName);
                        if (group.pendingCount() > 0) {
                            PendingMessagesSummary pendingMessagesSummary = redisTemplate.opsForStream().pending(key, groupName);
                            PendingMessages pendingMessages = redisTemplate.opsForStream().pending(key, group.groupName(), pendingMessagesSummary.getIdRange(), 100);
                            if (pendingMessages.size() > 0) {
                                for (PendingMessage pendingMessage : pendingMessages) {
                                    if (pendingMessage.getElapsedTimeSinceLastDelivery().toMillis() > redisStreamMessageQueueListener.timeout() * 1000) {
                                        final String recordId = pendingMessage.getId().getValue();
                                        streamMqLog.info("redis stream listen task. mqConfig: [{}] - stream: [{}] - group: [{}] - recordId: [{}] 发现消息超过限定时长[{}]s", redisStreamMessageQueueListener, key, group.groupName(), recordId, redisStreamMessageQueueListener.timeout());
                                        try {
                                            Map recordVal = null;
                                            // 查询消息内容
                                            List> readRes = redisTemplate.opsForStream().range(key, Range.closed(recordId, recordId), new RedisZSetCommands.Limit().count(1));
                                            if (readRes != null && readRes.size() > 0) {
                                                for (MapRecord v : readRes) {
                                                    if (StringUtils.equals(v.getId().getValue(), recordId)) {
                                                        recordVal = v.getValue();
                                                    }
                                                }
                                            }
                                            if (recordVal != null) {
                                                streamMqLog.info("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] 查询到消息的内容为[{}]", key, group.groupName(), recordId, JSON.toJSONString(recordVal));
                                                if (pendingMessage.getTotalDeliveryCount() <= redisStreamMessageQueueListener.maxDeliverTimes() && consumers.size() > 1) {
                                                    streamMqLog.info("redis stream listen task. stream: [{}] - group: [{}] - consumers: [{}]", key, group.groupName(), consumers);
                                                    // 移交其他消费者处理
                                                    String newConsumer = consumers.get(consumers.size() - 1).consumerName();
                                                    for (Iterator consumerIterator = consumers.stream().iterator(); ; consumerIterator.hasNext()) {
                                                        StreamInfo.XInfoConsumer curConsumer = consumerIterator.next();
                                                        if (StringUtils.equals(curConsumer.consumerName(), pendingMessage.getConsumerName())) {
                                                            break;
                                                        } else {
                                                            newConsumer = curConsumer.consumerName();
                                                        }
                                                    }
                                                    if (StringUtils.isNotBlank(newConsumer)) {
                                                        streamMqLog.info("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] 从消费者[{}]移交给消费者[{}]处理", key, group.groupName(), recordId, pendingMessage.getConsumerName(), newConsumer);
                                                        final String newConsumerFinal = newConsumer;
                                                        redisTemplate.execute(new RedisCallback>() {
                                                            @Override
                                                            public List doInRedis(RedisConnection connection) throws DataAccessException {
                                                                RedisStreamCommands.XClaimOptions claimOptions = RedisStreamCommands.XClaimOptions.minIdle(Duration.ofSeconds(1)).ids(pendingMessage.getId());
                                                                return connection.streamCommands().xClaim(key.getBytes(), groupName, newConsumerFinal, claimOptions.idle(Duration.ofSeconds(1)));
                                                            }
                                                        });
                                                    } else {
                                                        streamMqLog.warn("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] 移交消费者失败,未查询到有效的新消费者!", key, group.groupName(), recordId);
                                                    }
                                                } else {
                                                    // 记录日志或者放入配置好的队列中
                                                    if (redisStreamMessageQueueListener.enableDeadLetter()) {
                                                        String deadLetterKey = redisStreamMessageQueueListener.deadLetterKey();
                                                        if (StringUtils.isNotBlank(deadLetterKey) && !illegalDeadLetterKeys.contains(deadLetterKey) && redisStreamMessageQueueListener.deadLetterMaxSize() > 0) {
                                                            // 放入队列中
                                                            redisStreamMessageQueueStartService.coverSend(deadLetterKey, recordVal, new Integer(redisStreamMessageQueueListener.deadLetterMaxSize()).longValue());
                                                            streamMqLog.info("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] 放入死信队列[{}]中", key, group.groupName(), recordId, deadLetterKey);
                                                        } else {
                                                            // 报错违规
                                                            streamMqLog.warn("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] 放入死信队列[{}]失败!原因:死信队列为空或者违规套用死信队列,或未配置死信队列最大长度", key, group.groupName(), recordId, deadLetterKey);
                                                            if (streamMqLog.isDebugEnabled()) {
                                                                streamMqLog.info("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] - data: [{}]", key, group.groupName(), recordId, JSON.toJSONString(recordVal));
                                                            }
                                                        }
                                                    }
                                                }
                                            } else {
                                                streamMqLog.warn("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] 未查询到消息的内容", key, group.groupName(), recordId);
                                            }
                                        } catch (Exception e) {
                                            log.error("", e);
                                            streamMqLog.error("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] 处理数据失败,原因:{}", key, group.groupName(), recordId, e.getMessage());
                                        } finally {
                                            // ack数据
                                            Long ackLen = redisTemplate.opsForStream().acknowledge(key, groupName, recordId);
                                            streamMqLog.info("redis stream listen task. stream: [{}] - group: [{}] - recordId: [{}] 消息ack[{}]条", key, group.groupName(), recordId, ackLen);
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    /**
     * 添加监听
     *
     * @param event
     */
    @Override
    public void onApplicationEvent(ApplicationStartedEvent event) {
        Map beans = event.getApplicationContext().getBeansWithAnnotation(RedisStreamMessageQueueListener.class);
        List deadLetterKeys = new ArrayList<>();
        for (Object bean : beans.values()) {
            RedisStreamMessageQueueListener ca = bean.getClass().getAnnotation(RedisStreamMessageQueueListener.class);
            String group = ca.group();
            redisStreamMessageQueueStartService.listener(ca.value(), group, ca.type(), ca.autoAck(), ca.batchSize(), (StreamListener) bean);
            if (StringUtils.isNotBlank(group)) {
                ans.add(ca);
            }
            if (ca.enableDeadLetter() && StringUtils.isNotBlank(ca.deadLetterKey())) {
                deadLetterKeys.add(ca.deadLetterKey());
            }
        }
        // 获取违规死信队列(违规:死信队列又配置了死信队列)
        ans.forEach(v -> {
            if (v.enableDeadLetter() && StringUtils.isNotBlank(v.deadLetterKey()) && deadLetterKeys.contains(v.value())) {
                illegalDeadLetterKeys.add(v.value());
            }
        });
    }

    @Override
    public void afterPropertiesSet() {
        Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
            RLock lock = redissonClient.getLock(DISTUIBUTE_HANDLE_PENDING_DATA_LOCK + applicationName);
            boolean res = false;
            try {
                res = lock.tryLock(1, 10, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                log.error("", e);
            }
            if (!res) {
                return;
            }
            try {
                this.handlePendingData();
            } catch (Exception e) {
                log.error("", e);
            } finally {
                if (lock.isHeldByCurrentThread()) {
                    lock.unlock();
                } else {
                    log.warn("Lock not held by CurrentThread!!!");
                }
            }
        },
        0, 10, TimeUnit.SECONDS);
    }
}

package com.learning.mq.redis.service;

import org.springframework.data.redis.stream.StreamListener;

/**
 * 监听和发送方法
 */
public interface RedisStreamMessageQueueStartService {
    /**
     * 添加监听器
     * @param event
     * @param groupName
     * @param type
     * @param autoAck
     * @param batchSize
     * @param streamListener
     */
    void listener(String event, String groupName, Class type, boolean autoAck, int batchSize, StreamListener streamListener);

     void coverSend(String event, V val, Long maxSize);
}

package com.learning.mq.redis.service;

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.connection.stream.*;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.hash.ObjectHashMapper;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.stream.StreamListener;
import org.springframework.data.redis.stream.StreamMessageListenerContainer;
import com.learning.mq.redis.handler.CustomerErrorHandler;

import java.net.Inet4Address;
import java.net.UnknownHostException;
import java.time.Duration;

@Slf4j
public class RedisStreamMessageQueueStartServiceImpl implements RedisStreamMessageQueueStartService {

    private final long dataCenterId = getDataCenterId();

    private final RedisTemplate redisTemplate;

    long maxLen;

    public RedisStreamMessageQueueStartServiceImpl(RedisTemplate redisTemplate, Long maxLen) {
        this.redisTemplate = redisTemplate;
        this.maxLen = maxLen;
    }

    @Override
    public void listener(String event, String groupName, Class type, boolean autoAck, int batchSize, StreamListener streamListener) {
        // 如果是动态消费者组,则无需创建消费者组
        if (StringUtils.isNotBlank(groupName)) {
            createGroup(event, groupName);
        }
        startSubscription(event, groupName, type, autoAck, batchSize, streamListener);
    }

    @Override
    public  void coverSend(String event, V val, Long maxSize) {
        ObjectRecord record = StreamRecords.newRecord()
                .ofObject(val)
                .withId(RecordId.autoGenerate())
                .withStreamKey(event);
        redisTemplate.opsForStream().add(record);
        redisTemplate.opsForStream().trim(event, maxSize, true);
        if (log.isDebugEnabled()) {
            log.debug("event {} send content {}", event, val);
        }
    }


    private void startSubscription(String event, String groupName, Class type, boolean autoAck, int batchSize, StreamListener streamListener) {
        RedisConnectionFactory redisConnectionFactory = redisTemplate.getConnectionFactory();
        StreamMessageListenerContainer.StreamMessageListenerContainerOptions options = StreamMessageListenerContainer
                .StreamMessageListenerContainerOptions
                .builder()
                .pollTimeout(Duration.ofSeconds(1))
                .targetType(type)
                .batchSize(batchSize)
                .hashValueSerializer(RedisSerializer.string())
                .objectMapper(new ObjectHashMapper())
                .errorHandler(new CustomerErrorHandler())
                .build();
        StreamMessageListenerContainer listenerContainer = StreamMessageListenerContainer
                .create(redisConnectionFactory, options);
        if (StringUtils.isBlank(groupName)) {
            listenerContainer.receive(
                    StreamOffset.create(event, ReadOffset.lastConsumed()),
                    streamListener);
        } else {
            if (autoAck) {
                listenerContainer.receiveAutoAck(
                        Consumer.from(groupName, groupName + dataCenterId),
                        StreamOffset.create(event, ReadOffset.lastConsumed()),
                        streamListener);
            } else {
                listenerContainer.receive(
                        Consumer.from(groupName, groupName + dataCenterId),
                        StreamOffset.create(event, ReadOffset.lastConsumed()),
                        streamListener);
            }
        }
        listenerContainer.start();
    }

    private void createGroup(String event, String groupName) {
        try {
            String addGroupResult = redisTemplate.opsForStream().createGroup(event, ReadOffset.latest(), groupName);
            if (!StringUtils.equals(addGroupResult, "OK")) {
                log.error("STREAM - Group create failed");
            }
        } catch (Exception e) {
            log.info("STREAM - Group create failed. reason: exist group.");
        }
    }

    private static Long getDataCenterId() {
        try {
            String hostName = Inet4Address.getLocalHost().getHostName();
            int[] ints = StringUtils.toCodePoints(hostName);
            int sums = 0;
            for (int b : ints) {
                sums += b;
            }
            return (long) (sums % 32);
        } catch (UnknownHostException e) {
            // 如果获取失败,则使用随机数备用
            return RandomUtils.nextLong(0, 31);
        }
    }
}

1.9 监听类
package com.learning.mq.redis.listener;

import com.alibaba.fastjson.JSON;
import com.learning.mq.redis.annotation.RedisStreamMessageQueueListener;
import com.learning.mq.redis.dto.MessageDTO;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.connection.stream.ObjectRecord;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.stream.StreamListener;
import org.springframework.stereotype.Component;

@Component
@RedisStreamMessageQueueListener(value = "message-stream", group = "consumerGroup", type = String.class, timeout = 20, autoAck = true)
@AllArgsConstructor
@Slf4j
public class MessageListener implements StreamListener> {

    private final RedisTemplate redisTemplate;

    @Override
    public void onMessage(ObjectRecord message) {
        try {
            String stream = message.getStream();
            RecordId id = message.getId();
            String msg = message.getValue();
//            MessageDTO messageDTO = JSON.parseObject(msg, MessageDTO.class);
            log.info("消费的消息:{}", msg);
            //当是消费组消费时,如果不是自动ack,则需要在这个地方手动ack
//          redisTemplate.opsForStream().acknowledge("consumerGroup", message);
            //移除消息
            redisTemplate.opsForStream().delete(stream, id.toString());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

1.10 controller
package com.learning.mq.redis.controller;

import lombok.AllArgsConstructor;
import org.springframework.data.redis.connection.stream.ObjectRecord;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.connection.stream.StreamRecords;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

/**
 * @Description 消息发送
 */
@RestController
@RequestMapping("/message")
@AllArgsConstructor
public class MessageController {
    private RedisTemplate redisTemplate;

    @GetMapping("/save")
    public String save(@RequestParam String message){
//        try {
            String key = "message-stream";
//            MessageDTO messageDTO = new MessageDTO();
//            messageDTO.setContent(message);
//            ObjectRecord record = StreamRecords.newRecord()
//                    .in(key)
//                    .ofObject(messageDTO)
//                    .withId(RecordId.autoGenerate());
//            String msgId = redisTemplate.opsForStream().add(record).toString();
//            return msgId;
//        } catch (Exception e) {
//            e.printStackTrace();
//        }
//        return message;
        ObjectRecord record = StreamRecords.newRecord()
                .in(key)
                .ofObject(message)
                .withId(RecordId.autoGenerate());
        String msgId = redisTemplate.opsForStream().add(record).toString();
        return msgId;
    }
}

你可能感兴趣的:(redis,redis,消息队列)