Canal实时同步发送JSON数据示例代码
使用 Canal 需要安装Canal相关组件，安装步骤参阅：
Windows下Canal.admin-1.1.6安装部署 和 Windows下Canal.deployer-1.1.6安装部署
因合作需要，库中大半表数据需要实时同步给合作方，他们自己做分析处理用！
同时要求去除敏感数据和过滤一些不完整数据，最后决定用Canal来实时同步发送JSON数据给合作方，如下为示例代码：
主体接口代码：CanalClientService.java
import com.alibaba.otter.canal.protocol.CanalEntry;
/**
* Canal客户端服务接口
*
* @author songjianyong
*/
public interface CanalClientService extends Runnable {
/**
* 启动binlog日志监听客户端
*/
void startSlave();
/**
* 关闭binlog日志监听客户端
*/
void stopSlave();
/**
* 是否忽略库或表的同步
*
* @param entryHeader 条目标题
* @return boolean
*/
boolean ignoreTableSchemaOrName(CanalEntry.Header entryHeader);
}主体接口代码实现类：CanalClientServiceImpl.java
import cn.hutool.crypto.Mode;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import com.baomidou.mybatisplus.annotation.TableName;
import com.baomidou.mybatisplus.core.metadata.TableFieldInfo;
import com.baomidou.mybatisplus.core.metadata.TableInfo;
import com.baomidou.mybatisplus.core.toolkit.TableInfoHelper;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.reflections.Reflections;
import org.reflections.util.ConfigurationBuilder;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import org.springframework.util.ReflectionUtils;
import javax.annotation.Resource;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.net.URI;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Canal客户端服务接口实现类
*
* @author songjianyong
*/
@Slf4j
@Component
public class CanalClientServiceImpl implements CanalClientService, CommandLineRunner {
/**
* 是否关闭binlog日志监听客户端
*/
private volatile boolean stop;
/**
* 存放表和实体类的一一对应关系
*/
private final Map/*实体类*/> allTableEntity = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
private final ThreadFactory namedThreadFactory = new ThreadFactoryBuilder()
.setNameFormat("canal-client-%d").build();
private final ThreadPoolExecutor singleThreadPool = new ThreadPoolExecutor(1, 1,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(1), namedThreadFactory, new ThreadPoolExecutor.DiscardPolicy());
@Resource
private CanalProperties canalProperties;
@Resource
private ObjectMapper objectMapper;
@Resource
private CanalFeignClient canalFeignClient;
@Override
public void run() {
if (!Objects.equals(true, canalProperties.getEnableCanal())) {
log.warn("未开启Canal同步");
return;
}
if (!checkCanalProperties()) {
return;
}
stop = false;
SocketAddress address = new InetSocketAddress(canalProperties.getHostName(), canalProperties.getPort());
CanalConnector connector = CanalConnectors.newSingleConnector(address, canalProperties.getDestination(), canalProperties.getUsername(), canalProperties.getPassword());
int batchSize = (Objects.isNull(canalProperties.getBatchSize()) || canalProperties.getBatchSize() <= 0) ? 1000 : canalProperties.getBatchSize();
long emptyCount = 0;
long batchId = -1;
try {
connector.connect();
connector.subscribe(".*\\..*");
connector.rollback();
log.warn("Canal同步已启动");
while (!stop) {
// 获取指定数量的数据
Message message = connector.getWithoutAck(batchSize);
batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
emptyCount++;
if (Objects.equals(emptyCount % 60L, 0L)) {
log.warn("空闲 次数 : {}", emptyCount);
}
try {
TimeUnit.SECONDS.sleep(1L);
} catch (InterruptedException e) {
log.error(e.getMessage(), e);
}
} else {
emptyCount = 0;
log.info("message[batchId={},size={}]", batchId, size);
List values = parseEntry(message.getEntries());
deliverData(values);
}
// 提交确认
connector.ack(batchId);
}
} catch (Exception e) {
// 处理失败, 回滚数据
connector.rollback(batchId);
log.warn("Canal同步异常已停止");
connector.disconnect();
} finally {
log.warn("Canal同步已停止");
connector.disconnect();
}
}
/**
* 传送数据至合作方
*
* @param values 值
*/
void deliverData(List values) throws JsonProcessingException {
for (String deliverUrl : canalProperties.getDeliverUrls()) {
URI uri = URI.create(deliverUrl);
ResponseVO response = canalFeignClient.deliverData(uri, canalProperties.getReferer(), values);
Object[] args = {deliverUrl, System.lineSeparator(),
objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(response)};
log.info("Canal数据传递至[{}]结果：{}{}", args);
}
}
/**
* 解析条目数据
*
* @param entries 条目
* @return {@link List}<{@link Object}>
*/
private List parseEntry(List entries) throws JsonProcessingException, InstantiationException, IllegalAccessException {
List list = new ArrayList<>(entries.size());
for (CanalEntry.Entry entry : entries) {
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry, e);
}
CanalEntry.EventType eventType = rowChange.getEventType();
CanalEntry.Header entryHeader = entry.getHeader();
if (ignoreTableSchemaOrName(entryHeader)) {
continue;
}
Object[] args = {entryHeader.getLogfileName(), entryHeader.getLogfileOffset(),
entryHeader.getSchemaName(), entryHeader.getTableName(), eventType};
log.info("监听到binlog[{}:{}] , 表[{}.{}] , 事件 : {}", args);
Class clazz = allTableEntity.get(entryHeader.getTableName());
if (Objects.isNull(clazz)) {
log.warn("表对应实体类不存在：{}.{}", entryHeader.getSchemaName(), entryHeader.getTableName());
continue;
}
log.info("库表对应的实体类：[{}.{}] -> [{}]", entryHeader.getSchemaName(), entryHeader.getTableName(), clazz.getName());
buildData(list, rowChange, clazz);
}
return list;
}
/**
* 构建数据：JSON
*
* @param list 数据集合
* @param rowChange 变化记录
* @param clazz clazz
* @throws InstantiationException 实例化异常
* @throws IllegalAccessException 非法访问异常
* @throws JsonProcessingException json处理异常
*/
private void buildData(List list, CanalEntry.RowChange rowChange, Class clazz) throws InstantiationException, IllegalAccessException, JsonProcessingException {
CanalEntry.EventType eventType = rowChange.getEventType();
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
if (Objects.equals(eventType, CanalEntry.EventType.DELETE)) {
Object value = newInstance(clazz, rowData.getBeforeColumnsList());
add(list, value);
continue;
}
if (Objects.equals(eventType, CanalEntry.EventType.INSERT)) {
Object value = newInstance(clazz, rowData.getAfterColumnsList());
add(list, value);
continue;
}
if (Objects.equals(eventType, CanalEntry.EventType.UPDATE)) {
List afterColumnsList = rowData.getAfterColumnsList();
Object value = newInstance(clazz, afterColumnsList);
add(list, value);
}
}
}
/**
* 添加：忽略敏感数据
*
* @param list 列表
* @param value 数据
*/
void add(List list, Object value) throws JsonProcessingException {
if (Objects.isNull(value)) {
return;
}
// 数据敏感不添加到发送集合中，直接返回
if(sensitive(value)){
return;
}
String json = objectMapper.writeValueAsString(value);
// 数据加密处理
String token = UUID.randomUUID().toString();
CanalDeliverData canalDeliverData = new CanalDeliverData();
canalDeliverData.setDataType(CanalDataTypeEnum.JSON.name());
canalDeliverData.setToken(AesUtils.encryptHex(token, Mode.CBC));
canalDeliverData.setData(AesUtils.encryptHex(json, Mode.CBC));
list.add(canalDeliverData);
}
/**
* 反射方式构建实例对象：此处利用了mybatis-plus开源组件中表和实体类一一对应关系，保证数据字段的一致性
*
* @param clazz 表对应实体类
* @param columns 列数据
* @return {@link Object}
*/
private Object newInstance(Class clazz, List columns) throws InstantiationException, IllegalAccessException {
TableInfo tableInfo = TableInfoHelper.getTableInfo(clazz);
List fieldList = tableInfo.getFieldList();
Object value = clazz.newInstance();
long countKey = columns.stream().filter(CanalEntry.Column::getIsKey).count();
if (!Objects.equals(1L, countKey)) {
log.error("表主键数量必须是1才能同步：[{}:{}]", tableInfo.getTableName(), countKey);
return null;
}
for (CanalEntry.Column column : columns) {
if (!column.hasValue()) {
continue;
}
if (column.getIsKey() && StringUtils.isBlank(tableInfo.getKeyProperty())) {
log.warn("表主键字段无对应类属性：[{}.{}]", tableInfo.getTableName(), column.getName());
continue;
}
if (column.getIsKey() && StringUtils.isNotBlank(tableInfo.getKeyProperty())) {
// 设置主键属性值
setValue(value, tableInfo.getKeyProperty(), column, clazz, tableInfo);
continue;
}
TableFieldInfo tableFieldInfo = fieldList.stream()
.filter(e -> StringUtils.equalsIgnoreCase(e.getColumn(), column.getName()))
.findFirst()
.orElse(null);
if (Objects.isNull(tableFieldInfo)) {
log.warn("表字段无对应类属性：[{}.{}]", tableInfo.getTableName(), column.getName());
continue;
}
String property = tableFieldInfo.getProperty();
// 设置非主键属性值
setValue(value, property, column, clazz, tableInfo);
}
return value;
}
/**
* 反射方式设置对象属性值
*
* @param target 目标对象
* @param property 目标对象属性
* @param column 列数据
* @param clazz 目标对象类型
* @param tableInfo 表信息
*/
void setValue(Object target, String property, CanalEntry.Column column, Class clazz, TableInfo tableInfo) {
Field field = ReflectionUtils.findField(clazz, property);
if (Objects.isNull(field)) {
log.warn("类无此属性：{}.{}", clazz.getName(), property);
return;
}
String methodName = String.format("set%s", StringUtils.capitalize(property));
Method method = ReflectionUtils.findMethod(clazz, methodName, field.getType());
if (Objects.isNull(method)) {
log.warn("表字段无映射方法：{}.{}", tableInfo.getTableName(), column.getName());
return;
}
ReflectionUtils.invokeMethod(method, target, convert(field.getType(), column.getValue()));
}
/**
* 转换值
*
* @param type 值类型
* @param value 值
* @return {@link Object}
*/
Object convert(Class type, String value) {
if (Objects.equals(type, String.class)) {
return value;
}
if (Objects.equals(type, Integer.class) || Objects.equals(type, int.class)) {
return Integer.parseInt(value);
}
if (Objects.equals(type, Long.class) || Objects.equals(type, long.class)) {
return Long.parseLong(value);
}
if (Objects.equals(type, Date.class)) {
String[] parsePatterns = {"yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd"};
try {
return DateUtils.parseDate(value, parsePatterns);
} catch (ParseException e) {
log.error(e.getMessage(), e);
}
}
throw new RuntimeException(String.format("未处理类型：%s", type.getName()));
}
@Override
public void startSlave() {
if (singleThreadPool.getPoolSize() >= singleThreadPool.getCorePoolSize()) {
log.warn("Canal binlog 日志监听运行中，本次启动忽略！");
return;
}
singleThreadPool.execute(this);
}
@Override
public void stopSlave() {
stop = true;
}
/**
* 构建表和实体类的一一对应关系
*/
private void allTableEntity() {
if (!allTableEntity.isEmpty()) {
return;
}
Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage("com.coocaa"));
Set> classes = reflections.getTypesAnnotatedWith(TableName.class);
for (Class clazz : classes) {
TableName annotation = clazz.getAnnotation(TableName.class);
if (Objects.isNull(annotation)) {
continue;
}
allTableEntity.put(annotation.value(), clazz);
}
}
@Override
public boolean ignoreTableSchemaOrName(CanalEntry.Header entryHeader) {
String[] includeTableSchemas = canalProperties.getIncludeTableSchemas();
if (ArrayUtils.isNotEmpty(includeTableSchemas)
&& Arrays.stream(includeTableSchemas).noneMatch(e -> StringUtils.equalsIgnoreCase(e, entryHeader.getSchemaName()))) {
return true;
}
String[] excludeTableNames = canalProperties.getExcludeTableNames();
if (ArrayUtils.isEmpty(excludeTableNames)) {
return false;
}
for (String excludeTableName : excludeTableNames) {
String[] tableSchemaOrName = StringUtils.split(excludeTableName, ',');
if (tableSchemaOrName.length > 2) {
log.error("表配置格式错误：{}", excludeTableName);
return false;
}
if (Objects.equals(1, tableSchemaOrName.length)
&& StringUtils.equalsIgnoreCase(excludeTableName, entryHeader.getTableName())) {
log.warn("表的数据同步已忽略：{}", excludeTableName);
return true;
}
if (Objects.equals(2, tableSchemaOrName.length)
&& StringUtils.equalsIgnoreCase(tableSchemaOrName[0], entryHeader.getSchemaName())
&& StringUtils.equalsIgnoreCase(tableSchemaOrName[1], entryHeader.getTableName())) {
log.warn("库表的数据同步已忽略：{}.{}", tableSchemaOrName[0], tableSchemaOrName[1]);
return true;
}
}
return false;
}
/**
* 检查Canal配置
*/
boolean checkCanalProperties() {
if (StringUtils.isBlank(canalProperties.getHostName())) {
log.error("Canal 服务端主机地址未配置");
return false;
}
if (Objects.isNull(canalProperties.getPort())) {
log.error("Canal 服务端主机端口未配置");
return false;
}
if (StringUtils.isBlank(canalProperties.getDestination())) {
log.error("Canal 创建单链接的客户端链接目的地未配置");
return false;
}
if (StringUtils.isBlank(canalProperties.getUsername())) {
log.error("Canal MySQL slave 账号未配置");
return false;
}
if (ArrayUtils.isEmpty(canalProperties.getDeliverUrls())) {
log.error("Canal 数据投递地址未配置");
return false;
}
if (StringUtils.isBlank(canalProperties.getReferer())) {
log.error("Canal 数据投递来源地址未配置");
return false;
}
return true;
}
@Override
public void run(String... args) throws Exception {
// 项目启动时自动开启Canal同步
startSlave();
allTableEntity();
}
} 辅助配置类：CanalProperties.java
import com.alibaba.nacos.api.config.ConfigType;
import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties;
import lombok.Data;
import org.springframework.context.annotation.Configuration;
/**
* Canal同步配置属性类
*
* @author songjianyong
*/
@Data
@Configuration
@NacosConfigurationProperties(groupId = "groupId", dataId = "dataId", prefix = "canal", autoRefreshed = true, type = ConfigType.YAML)
public class CanalProperties {
/**
* 是否开启Canal同步
*/
private Boolean enableCanal;
/**
* the Host name
*/
private String hostName;
/**
* The port number，如：11111
*/
private Integer port;
/**
* 创建单链接的客户端链接目的地
*/
private String destination;
/**
* MySQL slave 账号
*/
private String username;
/**
* MySQL slave 密码
*/
private String password;
/**
* 获取指定数量的数据，默认1000
*/
private Integer batchSize;
/**
* 指定需要同步的库名
*/
private String[] includeTableSchemas;
/**
* 指定不需要同步的表名，格式：[库名.]表名
*/
private String[] excludeTableNames;
/**
* 数据投递来源地址
*/
private String referer;
/**
* 数据投递地址
*/
private String[] deliverUrls;
}其中类方法 CanalClientServiceImpl.newInstance 中利用了mybatis-plus 开源组件中表和实体类一一对应关系，对构建JSON实体类省去大量代码，否则若每个表数据手动构建，一旦新增新的表数据，又要不断累加代码了！
那么一旦数据实时同步过程中出了问题，表中数据同步到哪一行了呢，如何续传呢？因业务的特殊性，这里无需考虑此类问题！
但通常业务来说，数据传送过程中出了问题，一定要修复后重新续传的，那就需要额外提供手段了，此处不做深究！