基于redis、quartz的可重试的异步通知实现

1、数据库表设计

CREATE TABLE `crl_notify_record` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '自增主键',
  `notice_owner` varchar(30)  NOT NULL COMMENT '通知发起方的简称',
  `notice_owner_serial` varchar(80)  NOT NULL COMMENT '通知发起序列号',
  `notice_accepter` varchar(30)  NOT NULL COMMENT '通知接收方简称',
  `notify_url` varchar(300)  NOT NULL COMMENT '通知地址,http(s)://……',
  `notify_content` text  COMMENT '通知内容:明文json串',
  `notify_send_content` text  COMMENT '通知发送内容,有可能加密',
  `retry_count` int NOT NULL DEFAULT '20' COMMENT '通知重试次数,默认值：20',
  `notify_count` int NOT NULL DEFAULT '0' COMMENT '已通知次数',
  `last_notify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '最近处理时间,初始值与“下次通知时间”一致，每次拉取处理时变更。（可以推断出与预计处理时间的差距）',
  `next_notify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '下次通知时间,计算公式暂定：当前时间 + (n+5)*n^2.（n的单位：秒）',
  `delay_time` int NOT NULL DEFAULT '0' COMMENT '处理延时,单位：秒,处理延时，即“当前时间-下次通知时间”，用来看通知时效性',
  `send_status` varchar(20)  NOT NULL DEFAULT 'INIT' COMMENT '通知送达状态(http响应码为200则认为通知成功）.初始化：INIT；处理中：PROCESSING；成功：SUCCESS;未知：UNKNOWN;待处理：PRE_PROCESSING',
  `ret_type` varchar(10)  NOT NULL DEFAULT 'RETCON' COMMENT '通知方对通知回执的需求类型.无需回执（默认）：NONE；通知送达回执(接收方http响应200)：SENDED；通知应答回执（接收方应答的信息）：RETCON',
  `ret_content` varchar(512)  DEFAULT NULL COMMENT '通知接收方返回的通知应答内容',
  `notice_priority` int(11) NOT NULL DEFAULT '0' COMMENT '默认值：0，数值越大优先级越高',
  `created_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `modified_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uniq_notice_owner` (`notice_owner`,`notice_owner_serial`) USING BTREE,
  KEY `idx__query` (`send_status`,`next_notify_time`,`notice_priority`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='统一通知记录表';

2、代码逻辑：

     2.1 加重试机制的分布式锁

     2.2 扫描状态为初始化或待执行且发送时间小于当前时间的100条记录，按优先级排序。

     2.3  批量更新状态为处理中

     2.4  调用消息发送方法，更新通知记录最终状态。若有剩余次数则更为待处理，若没有则更新为未知，成功则更新为成功。

     2.5  整个任务每五分钟执行一次，可由quartz表达式配置

import com.alibaba.fastjson.JSON;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.util.CollectionUtils;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;

/**
 * 统一异步消息处理类
 */
public class CrlNotifyDispatcherJob {
    private static final Logger logger = LoggerFactory.getLogger(CrlNotifyDispatcherJob.class);

    @Resource
    private CrlNotifyRecordDao crlNotifyRecordDao;

    @Resource
    private CacheBizService cacheClient;

    //重复尝试占位的次数
    private static int max_sleep_count = 10;

    //每次占位的间隔时间ms
    private static int max_sleep_time = 1 * 1000;

    //查询数量
    private static int select_num = 100;

    //缓存占位过期时间
    private static int expire_time = 30;

    @Resource(name = "crlTaskQuartzRulesService")
    private CrlTaskQuartzRulesService crlTaskQuartzRulesService;

    @Resource
    private CrlHttpPostFacade crlHttpPostFacade;

    public synchronized void doDispatcherJob() {
        logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|start...");
        boolean isSetnx = false;
        try {
            //1.redis占位//这种占位的方式同一时刻只有一台机器能执行...瓶颈！
            if (!setnx(1)) {
                logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|redis占位失败");
                return;
            }
            isSetnx = true;
            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|查询开关配置开始|jobCode:" + JobTypeEnum.CrlNotifyDispatcherJob.getJobCode());
            CrlTaskQuartzRules crlTaskQuartzRules = crlTaskQuartzRulesService.selectByTaskCode(JobTypeEnum.CrlNotifyDispatcherJob.getJobCode());
            if (crlTaskQuartzRules == null || SwitchStatusEnum.OFF.getCode().equals(crlTaskQuartzRules.getSwitchStatus())) {
                logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|查询开关配置失败");
                return;
            }
            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|查询开关配置结束|jobCode:" + JobTypeEnum.CrlNotifyDispatcherJob.getJobCode() + crlTaskQuartzRules);
            //2.获取初始化init/待处理PRE_PROCESSING状态的数据，按照权重排序
            List statusList = Lists.newArrayList();
            statusList.add(NotifySendStatusEnum.INIT.getCode());
            statusList.add(NotifySendStatusEnum.PRE_PROCESSING.getCode());
            List notifyRecordList = crlNotifyRecordDao.selectHandleList(statusList, select_num);
            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|notifyRecordList" + JSON.toJSONString(notifyRecordList));
            if (CollectionUtils.isEmpty(notifyRecordList)) {
                logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|没有数据|end...");
                return;
            }
            //3.批量更新状态
            List notifyRecordNoList = Lists.transform(notifyRecordList, new Function() {
                @Override
                public Long apply(CrlNotifyRecord notifyRecord) {
                    return notifyRecord.getId();
                }
            });
            crlNotifyRecordDao.updateStatusByIds(notifyRecordNoList, NotifySendStatusEnum.PROCESSING.getCode());
            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|批量更新状态成功|notifyRecordNoList：" + JSON.toJSONString(notifyRecordNoList));
            //4.单个处理和计算和发消息
            for (CrlNotifyRecord notifyRecord : notifyRecordList) {
                String respStatus = "";
                //定义要赋值的NotifyRecord对象，先赋值无论成败都需要更新的属性
                CrlNotifyRecord upNotifyRecord = new CrlNotifyRecord();
                upNotifyRecord.setId(notifyRecord.getId());
                upNotifyRecord.setNotifyCount(notifyRecord.getNotifyCount() + 1);
                upNotifyRecord.setLastNotifyTime(new Date());
                upNotifyRecord.setNextNotifyTime(new Date(System.currentTimeMillis() + ((notifyRecord.getNotifyCount() + 5) * notifyRecord.getNotifyCount() * notifyRecord.getNotifyCount() * 1000)));
                upNotifyRecord.setDelayTime(new Long(System.currentTimeMillis() - notifyRecord.getNextNotifyTime().getTime()).intValue() / 1000);
                upNotifyRecord.setModifiedDate(new Date());
                try {
                     CrlCommonRequest crlCommonRequest=new CrlCommonRequest();
                    CrlHttpPostRequestDto crlHttpPostRequestDto=new CrlHttpPostRequestDto();
                    crlHttpPostRequestDto.setHttpUrl(notifyRecord.getNotifyUrl());
                    crlHttpPostRequestDto.setJsonStr(notifyRecord.getNotifySendContent());
                    crlCommonRequest.setRequestData(crlHttpPostRequestDto);
                    logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|调用通知接口开始|crlCommonRequest:" + crlCommonRequest);
                    CrlCommonResponse crlCommonResponse= crlHttpPostFacade.doHttpPost(crlCommonRequest);
                    if(!crlCommonResponse.isSuccess()){
                        logger.warn("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|调用通知接口没有成功|crlCommonRequest:" + crlCommonRequest);
                        continue;
                    }
                    logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|调用通知接口结束|crlCommonRequest:" + crlCommonRequest+"|crlCommonResponse:"+crlCommonResponse);
                    CrlBizCommonResponse crlBizCommonResponse=crlCommonResponse.getResData();
                    if(RespCodeEnum.HTTP_POST_SUCCESS.getCode().equals(crlBizCommonResponse.getResBizCode())){
                        respStatus = NotifySendStatusEnum.SUCCESS.getCode();
                        logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|发送加密加签后的通知返回成功|crlCommonRequest:" + crlCommonRequest+"|crlBizCommonResponse:" + crlBizCommonResponse);
                        if (NotifyRetTypeEnum.SENDED.getCode().equals(notifyRecord.getRetType())) {
                            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|通知送达回执(接收方http响应200)|crlCommonRequest:" + crlCommonRequest+"|crlBizCommonResponse:" + crlBizCommonResponse);
                            upNotifyRecord.setRetContent(RespCodeEnum.HTTP_POST_SUCCESS.getDesc());
                        } else if (NotifyRetTypeEnum.RETCON.getCode().equals(notifyRecord.getRetType())) {
                            //需要通知回执，则为upNotifyRecord的回执内容赋值，若有业务逻辑变更请修改
                            upNotifyRecord.setRetContent(crlBizCommonResponse.getResBizMsg());
                            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|通知应答回执（接收方应答的信息）|crlCommonRequest:" + crlCommonRequest+"|crlBizCommonResponse:" + crlBizCommonResponse);
                        } else {
                            //默认不需要通知回执
                            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|无需回执（默认）|crlCommonRequest:" + crlCommonRequest+"|crlBizCommonResponse:" + crlBizCommonResponse);
                        }
                    }else if(notifyRecord.getNotifyCount() > notifyRecord.getRetryCount()){
                        respStatus = NotifySendStatusEnum.UNKNOWN.getCode();
                    }else{
                        respStatus = NotifySendStatusEnum.PRE_PROCESSING.getCode();
                    }
                } catch (Exception e) {
                    logger.warn("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务失败" + LogConstant.SYS_SCHEDULE_EXCEPTION + "|Exception:" + ExceptionUtils.getStackTrace(e));
                    respStatus = NotifySendStatusEnum.PRE_PROCESSING.getCode();
                }
                //将不同的响应状态赋值然后更新通知记录
                upNotifyRecord.setSendStatus(respStatus);
                int upCount = crlNotifyRecordDao.updateByPrimaryKeySelective(upNotifyRecord);
                logger.info("DispatcherService|doDispatcherJob|异步结果通知任务|更新通知记录成功|upNotifyRecord：" + JSON.toJSONString(upNotifyRecord) + "|upCount" + upCount);
                if (notifyRecord.getNotifyCount() > notifyRecord.getRetryCount()) {
                    logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务|通知次数已经达到最大次数|NotifyCount:" + notifyRecord.getNotifyCount() + "|RetryCount:" + notifyRecord.getRetryCount());
                }
            }
        } catch (Exception e) {
            logger.error("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务异常" + LogConstant.SYS_SCHEDULE_EXCEPTION + "|Exception:" + ExceptionUtils.getStackTrace(e));
        } finally {

            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|异步结果通知任务结束|清除占位");
            if (isSetnx) {
                cacheClient.del(CacheConstant.NOTIFY_DISPATCHER_CAS_KEY);
            }
        }
    }

    //递归占位
    private boolean setnx(int count) throws Exception {
        if (count > max_sleep_count) {
            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|setnx|异步结果通知任务|递归同步占位失败" + count);
            return false;
        }
        Long redisCAS = cacheClient.setnx(CacheConstant.NOTIFY_DISPATCHER_CAS_KEY, "1");
        if (redisCAS == null || (!redisCAS.equals(new Long(1)))) {//占位失败
            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|setnx|异步结果通知任务|递归同步占位第count次失败|redisCAS:" + redisCAS + "|count:" + count);
            Thread.sleep(max_sleep_time);
            return setnx(count + 1);
        } else {
            cacheClient.expire(CacheConstant.NOTIFY_DISPATCHER_CAS_KEY, expire_time);
            logger.info("CrlNotifyDispatcherJob|doDispatcherJob|setnx|异步结果通知任务|递归同步占位成功|redisCAS:" + redisCAS + "|count:" + count);
            return true;
        }
    }

}