Canal + RocketMQ 同步 MySQL 数据到 Redis
解决缓存和数据库一致性问题
一般来说，缓存中的数据没什么问题，但是数据库更新后，就容易出现缓存（Redis）和数据库（MySQL）间的数据一致性问题。由于写和读是并发的，没法保证顺序，就会出现缓存和数据库的数据不一致的问题
Canal工作原理
- canal模拟mysql slave的交互协议，伪装自己为mysql slave，向mysql master发送dump协议
- mysql master收到dump请求，开始推送binary log给slave(也就是canal)
- canal解析binary log对象(原始为byte流)
Canal的安装与配置（略）
Java操作开始
pom依赖
org.apache.rocketmq
rocketmq-spring-boot-starter
2.2.3
com.alibaba
fastjson
javax.persistence
persistence-api
1.0.2
provided
com.baomidou
mybatis-plus-boot-starter
3.5.1
com.alibaba.otter
canal.client
1.1.4
Article.java
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.extension.activerecord.Model;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.Accessors;
/**
* @ClassName Article
* @Description TODO
* @Author fan
* @Date 2024/1/29 11:59
* @Version 1.0
*/
@Data
@EqualsAndHashCode(callSuper = false)
@Accessors(chain = true)
public class Article extends Model {
@TableId(value = "id", type = IdType.AUTO)
private Long id;
private String content;
} SqlType.enum
/**
* @ClassName SqlType Canal监听SQL类型
* @Description TODO
* @Author fan
* @Date 2024/1/29 11:50
* @Version 1.0
*/
@SuppressWarnings("AlibabaEnumConstantsMustHaveComment")
public enum SqlType {
INSERT("INSERT", "插入"),
UPDATE("UPDATE", "更新"),
DELETE("DELETE", "删除");
private final String type;
private final String name;
SqlType(String type, String name) {
this.type = type;
this.name = name;
}
public String getType() {
return this.type;
}
public String getName() {
return this.name;
}
}
定义接口CanalSyncService
package com.example.myrocketmqboot.service;
import com.alibaba.otter.canal.protocol.FlatMessage;
import java.util.Collection;
/**
* @ClassName CanalSyncService Canal同步服务
* @Description TODO
* @Author fan
* @Date 2024/1/29 14:13
* @Version 1.0
*/
public interface CanalSyncService {
/**
* 处理数据
*
* @param flatMessage CanalMQ数据
*/
void process(FlatMessage flatMessage);
/**
* DDL语句处理
*
* @param flatMessage CanalMQ数据
*/
void ddl(FlatMessage flatMessage);
/**
* 插入
*
* @param list 新增数据
*/
void insert(Collection list);
/**
* 更新
*
* @param list 更新数据
*/
void update(Collection list);
/**
* 删除
*
* @param list 删除数据
*/
void delete(Collection list);
}
ArticleCanalRocketMqRedisService实现CanalSyncService接口
package com.example.myrocketmqboot.service.impl;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.annotation.TableId;
import com.example.myrocketmqboot.listener.SqlType;
import com.example.myrocketmqboot.service.CanalSyncService;
import lombok.extern.slf4j.Slf4j;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.google.common.collect.Sets;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.util.ReflectionUtils;
import javax.annotation.Resource;
import java.lang.reflect.Field;
import java.lang.reflect.ParameterizedType;
import java.util.*;
/**
* @ClassName AbstractCanalRocketMqRedisService 抽象Canal-RocketMQ通用处理服务
* @Description TODO
* @Author fan
* @Date 2024/1/29 14:14
* @Version 1.0
*/
@Slf4j
public abstract class ArticleCanalRocketMqRedisService implements CanalSyncService {
@Resource
private RedisTemplate redisTemplate;
@Resource
private com.fan.li.myspringboot.util.RedisClient redisUtils;
private Class classCache;
/**
* 获取Model名称
*
* @return Model名称
*/
protected abstract String getModelName();
/**
* 处理数据
*
* @param flatMessage CanalMQ数据
*/
@Override
public void process(FlatMessage flatMessage) {
if (flatMessage.getIsDdl()) {
ddl(flatMessage);
return;
}
Set data = getData(flatMessage);
if (SqlType.INSERT.getType().equals(flatMessage.getType())) {
insert(data);
}
if (SqlType.UPDATE.getType().equals(flatMessage.getType())) {
update(data);
}
if (SqlType.DELETE.getType().equals(flatMessage.getType())) {
delete(data);
}
}
/**
* DDL语句处理
*
* @param flatMessage CanalMQ数据
*/
@Override
public void ddl(FlatMessage flatMessage) {
}
/**
* 插入
*
* @param list 新增数据
*/
@Override
public void insert(Collection list) {
insertOrUpdate(list);
}
/**
* 更新
*
* @param list 更新数据
*/
@Override
public void update(Collection list) {
insertOrUpdate(list);
}
/**
* 删除
*
* @param list 删除数据
*/
@Override
public void delete(Collection list) {
Set keys = Sets.newHashSetWithExpectedSize(list.size());
for (T data : list) {
keys.add(getWrapRedisKey(data));
}
for (String key : keys) {
redisUtils.deleteKey(key);
}
}
/**
* 插入或者更新redis
*
* @param list 数据
*/
@SuppressWarnings("unchecked")
private void insertOrUpdate(Collection list) {
redisTemplate.executePipelined((RedisConnection redisConnection) -> {
for (T data : list) {
String key = getWrapRedisKey(data);
// 序列化key
byte[] redisKey = redisTemplate.getKeySerializer().serialize(key);
// 序列化value
byte[] redisValue = redisTemplate.getValueSerializer().serialize(data);
redisConnection.set(Objects.requireNonNull(redisKey), Objects.requireNonNull(redisValue));
}
return null;
});
}
/**
* 封装redis的key
*
* @param t 原对象
* @return key
*/
protected String getWrapRedisKey(T t) {
return getModelName() + ":" + getIdValue(t);
}
/**
* 获取类泛型
*
* @return 泛型Class
*/
@SuppressWarnings("unchecked")
protected Class getTypeArgument() {
if (classCache == null) {
classCache = (Class) ((ParameterizedType) this.getClass().getGenericSuperclass()).getActualTypeArguments()[0];
}
return classCache;
}
/**
* 获取Object标有@TableId注解的字段值
*
* @param t 对象
* @return id值
*/
protected Object getIdValue(T t) {
Field fieldOfId = getIdField();
ReflectionUtils.makeAccessible(fieldOfId);
return ReflectionUtils.getField(fieldOfId, t);
}
/**
* 获取Class标有@TableId注解的字段名称
*
* @return id字段名称
*/
protected Field getIdField() {
Class clz = getTypeArgument();
Field[] fields = clz.getDeclaredFields();
for (Field field : fields) {
TableId annotation = field.getAnnotation(TableId.class);
if (annotation != null) {
return field;
}
}
log.error("PO类未设置@TableId注解");
return null;
}
/**
* 转换Canal的FlatMessage中data成泛型对象
*
* @param flatMessage Canal发送MQ信息
* @return 泛型对象集合
*/
protected Set getData(FlatMessage flatMessage) {
List> sourceData = flatMessage.getData();
Set targetData = Sets.newHashSetWithExpectedSize(sourceData.size());
for (Map map : sourceData) {
T t = JSON.parseObject(JSON.toJSONString(map), getTypeArgument());
targetData.add(t);
}
return targetData;
}
}
ArticleConsumer消费
package com.example.myrocketmqboot.consumer.article;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.example.myrocketmqboot.pojo.Article;
import com.example.myrocketmqboot.service.impl.AbstractCanalRocketMqRedisService;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.apache.rocketmq.spring.core.RocketMQPushConsumerLifecycleListener;
import org.springframework.stereotype.Service;
/**
* @ClassName ArticleConsumer
* @Description TODO
* @Author fan
* @Date 2024/1/29 14:19
* @Version 1.0
*/
@Slf4j
@Service
@RocketMQMessageListener(topic = "Consumer_Article", consumerGroup = "myArticle")
public class ArticleConsumer extends AbstractCanalRocketMqRedisService
implements RocketMQListener, RocketMQPushConsumerLifecycleListener {
@Override
public void onMessage(FlatMessage flatMessage) {
log.info("consumer message {}", flatMessage);
try {
process(flatMessage);
} catch (Exception e) {
log.warn(String.format("[onMessage_消费失败]", flatMessage), e);
throw new RuntimeException(e);
}
}
@Getter
private final String modelName = Article.class.getSimpleName();
@Override
public void prepareStart(DefaultMQPushConsumer consumer) {
consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
consumer.setConsumeTimestamp(UtilAll.timeMillisToHumanString3(System.currentTimeMillis()));
}
}
测试：
1、往article表插入一条数据
2、查看redis有该条数据即可！