Canal使用-基于读取binlog

1、https://github.com/alibaba/canal

2、示例代码

package net.wecash.o2ocanalclient.canal;

import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import net.wecash.o2ocanalclient.entity.OrderdetailDO;
import net.wecash.o2ocanalclient.entity.RcOrderdetailDO;
import net.wecash.o2ocanalclient.entity.RepaymentPlanDO;
import net.wecash.o2ocanalclient.entity.ThirdOrderDetailDO;
import net.wecash.o2ocanalclient.mappers.RcOrderdetailMapper;
import net.wecash.o2ocanalclient.utils.CanalUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Component;

import java.net.InetSocketAddress;
import java.util.List;
import java.util.Map;

/**
 * 数据导入 将orderdetail 和 pl_repayment_plan表的数据实时更新到rc_orderdetail表中
 * 利用canal读取数据库binlog文件
 * @Author:Luobh
 * @create: 2018-11-09
 **/
@Component
public class DataImportTask {

    Logger logger = LogManager.getLogger(this.getClass());
    @Autowired
    RcOrderdetailMapper rcOrderdetailMapper;
    @Value("${canal.zkServers}")
    String zkServers;
    @Value("${canal.destination}")
    String destination;
    @Autowired
    TaskExecutor taskExecutor;

    public void start() {
        taskExecutor.execute(new Runnable() {
            @Override
            public void run() {
                logger.info("data import start...zkServers->{},destination->{}",zkServers,destination);
                // 基于zookeeper动态获取canal server的地址，建立链接，其中一台server发生crash，可以支持failover
                CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(AddressUtils.getHostIp(), 11111),"example", "", "");
                //CanalConnector connector = CanalConnectors.newClusterConnector(zkServers, destination, "", "");
                int batchSize = 1000;
                try {
                    connector.connect();
                    //订阅关注的表
                    connector.subscribe("o2o.pl_repayment_plan_user,o2o.orderdetail,o2o.o2o_third_orderdetail");
                    while (true) {
                        long batchId = -1;
                        Message message = null;
                        try {
                            //获取指定数量的数据 不设置阻塞时间
                            message = connector.getWithoutAck(batchSize);
                            batchId = message.getId();
                            int size = message.getEntries().size();
                            if (batchId == -1 || size == 0) {
                                try {
                                    Thread.sleep(1000);
                                } catch (InterruptedException e) {
                                    logger.error("canal sleep error,", e);
                                }
                            } else {
                                logger.info("deal with message start,size->{}", size);
                                dealWithEntry(message.getEntries());
                            }
                             //提交确认
                             connector.ack(batchId);
                        } catch (Exception e) {
                            //失败后不回滚，记录下错误日志
                            logger.info("canal deal with message error,batchId->{},message->{}", batchId, message,e);
                        }
                    }
                } finally {
                    logger.info("connector disconnect... ");
                    connector.disconnect();
                }
            }
        });

    }

    private void dealWithEntry(List entrys) {
        for (CanalEntry.Entry entry : entrys) {
            try {
                CanalEntry.RowChange rowChage = null;
                try {
                    rowChage = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
                } catch (Exception e) {
                    throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                }
                logger.info("tableName->{},rowChage->{}",entry.getHeader().getTableName(),rowChage);
                //根据不同表来源的数据进行操作
                tableProxy(entry.getHeader().getTableName(),rowChage);
            } catch (Exception e) {
                //逐条记录错误信息，防止一条数据影响多条
                logger.error("dealWithEntry error,entry->{}",entry.toString(),e);
            }
        }
    }

    private void tableProxy(String tableName,CanalEntry.RowChange rowChage) {
        CanalEntry.EventType eventType = rowChage.getEventType();
        if ("orderdetail".equals(tableName)) {
            orderdetailHandler(rowChage,eventType);

        } else if ("pl_repayment_plan_user".equals(tableName)) {
            plRepaymentPlanHandler(rowChage,eventType);

        } else if ("o2o_third_orderdetail".equals(tableName)) {
            thirdOrderdetailHandler(rowChage,eventType);

        }
    }

    /**
     * 老贷后还款计划表
     * orderdetail
     * @param rowChage
     * @param eventType
     */
    private void orderdetailHandler(CanalEntry.RowChange rowChage,CanalEntry.EventType eventType) {
        for (CanalEntry.RowData rowData : rowChage.getRowDatasList()) {
            if (eventType == CanalEntry.EventType.INSERT || eventType == CanalEntry.EventType.UPDATE) {
                //插入 更新
                OrderdetailDO orderdetailDO = new OrderdetailDO();
                CanalUtil.columnListToDO(rowData.getAfterColumnsList(),orderdetailDO);
                orderDetailToRc(orderdetailDO);
            } else if (eventType == CanalEntry.EventType.DELETE) {
                //删除
                Long id = CanalUtil.getDelId(rowData.getBeforeColumnsList());
                rcOrderdetailMapper.deleteRcOrderdetailById(id);
            }
        }
    }

    /**
     * 新贷后还款计划表
     * pl_repayment_plan_user
     * @param rowChage
     * @param eventType
     */
    private void plRepaymentPlanHandler(CanalEntry.RowChange rowChage,CanalEntry.EventType eventType) {
        for (CanalEntry.RowData rowData : rowChage.getRowDatasList()) {
            if (eventType == CanalEntry.EventType.INSERT || eventType == CanalEntry.EventType.UPDATE) {
                //插入 更新
                RepaymentPlanDO repaymentPlanDO = new RepaymentPlanDO();
                CanalUtil.columnListToDO(rowData.getAfterColumnsList(),repaymentPlanDO);
                if (repaymentPlanDO.getPeriod()==null || repaymentPlanDO.getOrderId()==null) {
                    Map map = rcOrderdetailMapper.getPeriodAndOrderIdByPlOrderId(repaymentPlanDO.getPlOrderId());
                    if (map!=null) {
                        Integer period = map.get("period");
                        Integer orderId = map.get("order_id");
                        repaymentPlanDO.setPeriod(period);
                        repaymentPlanDO.setOrderId(Long.valueOf(orderId));
                    }
                }
                repaymentPlanUserToRc(repaymentPlanDO);
            } else if (eventType == CanalEntry.EventType.DELETE) {
                //删除
                Long id = CanalUtil.getDelId(rowData.getBeforeColumnsList());
                rcOrderdetailMapper.deleteRcOrderdetailById(id);
            }
        }
    }

    /**
     * 旖美表
     * o2o_third_orderdetail
     * @param rowChage
     * @param eventType
     */
    private void thirdOrderdetailHandler(CanalEntry.RowChange rowChage,CanalEntry.EventType eventType) {
        for (CanalEntry.RowData rowData : rowChage.getRowDatasList()) {
            if (eventType == CanalEntry.EventType.INSERT || eventType == CanalEntry.EventType.UPDATE) {
                //插入 更新
                ThirdOrderDetailDO thirdOrderDetailDO = new ThirdOrderDetailDO();
                CanalUtil.columnListToDO(rowData.getAfterColumnsList(),thirdOrderDetailDO);
                thirdOrderDetailToRc(thirdOrderDetailDO);
            } else if (eventType == CanalEntry.EventType.DELETE) {
                //删除
                Long id = CanalUtil.getDelId(rowData.getBeforeColumnsList());
                rcOrderdetailMapper.deleteRcOrderdetailById(id);
            }
        }
    }

    private void repaymentPlanUserToRc(RepaymentPlanDO planDO) {
        try {
            RcOrderdetailDO rcdo = new RcOrderdetailDO();
            BeanUtils.copyProperties(planDO, rcdo);
            rcdo.setOrderId(planDO.getOrderId());
            rcdo.setStatus(planDO.getBillStatus());
            rcdo.setRepayment(planDO.getPeriod() + "-" + planDO.getCurrentPeriod());
            rcdo.setRepaymentTime(planDO.getShouldRepayTime());
            rcdo.setPaymentDate(planDO.getRealRepayTime());
            rcdo.setStage(planDO.getEveryPrincipalInterest());
            rcdo.setPrincipal(planDO.getShouldRepayPrincipal());
            rcdo.setInterests(planDO.getShouldRepayInterest());
            rcdo.setIsOverdue(planDO.getOverdueSign());
            rcdo.setOverDay(planDO.getOverdueDays());
            rcdo.setRemainAmount(planDO.getRemainTotal());
            rcdo.setShouldAmount(planDO.getShouldRepayTotal());
            rcdo.setRealAmount(planDO.getRealRepayTotal());
            rcdo.setDemurrage(planDO.getShouldRepayOverdueFee());
            rcdo.setLsnum(planDO.getLsnum());
            rcdo.setPaytype(planDO.getRepayType());
            rcdo.setCompensateAmount(planDO.getChannelPayAmount());
            rcdo.setCompensateDate(planDO.getChannelPayTime());
            rcdo.setCreateTime(planDO.getCreateTime());
            rcdo.setUpdateTime(planDO.getUpdateTime());
            rcOrderdetailMapper.saveRcOrderdetail(rcdo);
        } catch (Exception e) {
            logger.error("repaymentPlanUserToRc error",e);
        }
    }

    private void orderDetailToRc(OrderdetailDO orderdetailDO) {
        try {
            RcOrderdetailDO rcdo = new RcOrderdetailDO();
            BeanUtils.copyProperties(orderdetailDO, rcdo);
            rcdo.setCreateTime(orderdetailDO.getCreateTime());
            rcdo.setUpdateTime(orderdetailDO.getUpdateTime());
            rcOrderdetailMapper.saveRcOrderdetail(rcdo);
        } catch (Exception e) {
            logger.error("orderDetailToRc error,",e);
        }
    }

    private void thirdOrderDetailToRc(ThirdOrderDetailDO thirdOrderDetailDO) {
        try {
            RcOrderdetailDO rcdo = new RcOrderdetailDO();
            BeanUtils.copyProperties(thirdOrderDetailDO, rcdo);
            rcdo.setId(thirdOrderDetailDO.getId()+3000000000000000000L);
            rcdo.setPaymentDate(thirdOrderDetailDO.getPaymentTime());
            rcdo.setCreateTime(thirdOrderDetailDO.getCreateTime());
            rcdo.setUpdateTime(thirdOrderDetailDO.getUpdateTime());
            rcOrderdetailMapper.saveRcOrderdetail(rcdo);
        } catch (Exception e) {
            logger.error("orderDetailToRc error,",e);
        }
    }
}

3、遇到的问题

问题1：column size is not match for table:o2o.pl_repayment_plan_user,52 vs 51

 

检查配置文件/canal.deployer-1.1.1/conf/canal.properties

canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml

以及

# table meta tsdb info
canal.instance.tsdb.enable=true
canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername=canal
canal.instance.tsdb.dbPassword=canal

检查 tsdb是否已经开启 

如果都已开启还是有这个问题 则需要清除canal对表结构的缓存

conf/example/h2.mv.db

问题2：找不到binlog日志文件

清空缓存

canal.deployer-1.1.1/conf/example/meta.dat