JAVA实现mysql的binlog监听(mysql-binlog-connector)

1. mysql-binlog-connector的简单使用

  • 引入依赖:
        
            com.github.shyiko
            mysql-binlog-connector-java
            0.16.1
        
  • JAVA的Demo
@Slf4j
public class TestBinlog {
    private static final ParserConfig snakeCase;

    static {
        snakeCase = new ParserConfig();
        snakeCase.propertyNamingStrategy = PropertyNamingStrategy.SnakeCase;
    }

    public static void main(String[] args) throws IOException {

        BinaryLogClient client = new BinaryLogClient("localhost", 3306, "root", "123qwe");
        EventDeserializer eventDeserializer = new EventDeserializer();
        //时间反序列化的格式
//        eventDeserializer.setCompatibilityMode(
//                EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG,
//                EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY
//        );
        client.setEventDeserializer(eventDeserializer);


        client.registerEventListener(new BinaryLogClient.EventListener() {

            @Override
            public void onEvent(Event event) {

                EventHeader header = event.getHeader();


                EventType eventType = header.getEventType();
                System.out.println("监听的事件类型:" + eventType);

                if (EventType.isWrite(eventType)) {
                    //获取事件体
                    WriteRowsEventData data = event.getData();
                    log.info(JSON.toJSONString(data));
                } else if (EventType.isUpdate(eventType)) {
                    UpdateRowsEventData data = event.getData();
                    log.info(JSON.toJSONString(data));
                } else if (EventType.isDelete(eventType)) {
                    DeleteRowsEventData data = event.getData();
                    log.info(JSON.toJSONString(data));
                }
            }
        });
        client.connect();
    }
}

注意点:

  1. client监听维度是数据库级别的监听;
  2. 可以监听表的DML语句和DDL语句;

2. mysql-binlog-connector的监听事件

源码位置:com.github.shyiko.mysql.binlog.event.EventType

public enum EventType {

    /**
     * Events of this event type should never occur. Not written to a binary log.
     */
    UNKNOWN,
    /**
     * A descriptor event that is written to the beginning of the each binary log file. (In MySQL 4.0 and 4.1,
     * this event is written only to the first binary log file that the server creates after startup.) This event is
     * used in MySQL 3.23 through 4.1 and superseded in MySQL 5.0 by {@link #FORMAT_DESCRIPTION}.
     */
    START_V3,
    /**
     * Written when an updating statement is done.
     */
    QUERY,
    /**
     * Written when mysqld stops.
     */
    STOP,
    /**
     * Written when mysqld switches to a new binary log file. This occurs when someone issues a FLUSH LOGS statement or
     * the current binary log file becomes larger than max_binlog_size.
     */
    ROTATE,
    /**
     * Written every time a statement uses an AUTO_INCREMENT column or the LAST_INSERT_ID() function; precedes other
     * events for the statement. This is written only before a {@link #QUERY} and is not used in case of RBR.
     */
    INTVAR,
    /**
     * Used for LOAD DATA INFILE statements in MySQL 3.23.
     */
    LOAD,
    /**
     * Not used.
     */
    SLAVE,
    /**
     * Used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
     */
    CREATE_FILE,
    /**
     * Used for LOAD DATA INFILE statements as of MySQL 4.0.
     */
    APPEND_BLOCK,
    /**
     * Used for LOAD DATA INFILE statements in 4.0 and 4.1.
     */
    EXEC_LOAD,
    /**
     * Used for LOAD DATA INFILE statements as of MySQL 4.0.
     */
    DELETE_FILE,
    /**
     * Used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
     */
    NEW_LOAD,
    /**
     * Written every time a statement uses the RAND() function; precedes other events for the statement. Indicates the
     * seed values to use for generating a random number with RAND() in the next statement. This is written only
     * before a {@link #QUERY} and is not used in case of RBR.
     */
    RAND,
    /**
     * Written every time a statement uses a user variable; precedes other events for the statement. Indicates the
     * value to use for the user variable in the next statement. This is written only before a {@link #QUERY} and
     * is not used in case of RBR.
     */
    USER_VAR,
    /**
     * A descriptor event that is written to the beginning of the each binary log file.
     * This event is used as of MySQL 5.0; it supersedes {@link #START_V3}.
     */
    FORMAT_DESCRIPTION,
    /**
     * Generated for a commit of a transaction that modifies one or more tables of an XA-capable storage engine.
     * Normal transactions are implemented by sending a {@link #QUERY} containing a BEGIN statement and a {@link #QUERY}
     * containing a COMMIT statement (or a ROLLBACK statement if the transaction is rolled back).
     */
    XID,
    /**
     * Used for LOAD DATA INFILE statements as of MySQL 5.0.
     */
    BEGIN_LOAD_QUERY,
    /**
     * Used for LOAD DATA INFILE statements as of MySQL 5.0.
     */
    EXECUTE_LOAD_QUERY,
    /**
     * This event precedes each row operation event. It maps a table definition to a number, where the table definition
     * consists of database and table names and column definitions. The purpose of this event is to enable replication
     * when a table has different definitions on the master and slave. Row operation events that belong to the same
     * transaction may be grouped into sequences, in which case each such sequence of events begins with a sequence
     * of TABLE_MAP events: one per table used by events in the sequence.
     * Used in case of RBR.
     */
    TABLE_MAP,
    /**
     * Describes inserted rows (within a single table).
     * Used in case of RBR (5.1.0 - 5.1.15).
     */
    PRE_GA_WRITE_ROWS,
    /**
     * Describes updated rows (within a single table).
     * Used in case of RBR (5.1.0 - 5.1.15).
     */
    PRE_GA_UPDATE_ROWS,
    /**
     * Describes deleted rows (within a single table).
     * Used in case of RBR (5.1.0 - 5.1.15).
     */
    PRE_GA_DELETE_ROWS,
    /**
     * Describes inserted rows (within a single table).
     * Used in case of RBR (5.1.16 - mysql-trunk).
     */
    WRITE_ROWS,
    /**
     * Describes updated rows (within a single table).
     * Used in case of RBR (5.1.16 - mysql-trunk).
     */
    UPDATE_ROWS,
    /**
     * Describes deleted rows (within a single table).
     * Used in case of RBR (5.1.16 - mysql-trunk).
     */
    DELETE_ROWS,
    /**
     * Used to log an out of the ordinary event that occurred on the master. It notifies the slave that something
     * happened on the master that might cause data to be in an inconsistent state.
     */
    INCIDENT,
    /**
     * Sent by a master to a slave to let the slave know that the master is still alive. Not written to a binary log.
     */
    HEARTBEAT,
    /**
     * In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
     * case there is code for handling it, but which can be ignored if it is not recognized.
     */
    IGNORABLE,
    /**
     * Introduced to record the original query for rows events in RBR.
     */
    ROWS_QUERY,
    /**
     * Describes inserted rows (within a single table).
     * Used in case of RBR (5.1.18+).
     */
    EXT_WRITE_ROWS,
    /**
     * Describes updated rows (within a single table).
     * Used in case of RBR (5.1.18+).
     */
    EXT_UPDATE_ROWS,
    /**
     * Describes deleted rows (within a single table).
     * Used in case of RBR (5.1.18+).
     */
    EXT_DELETE_ROWS,
    /**
     * Global Transaction Identifier.
     */
    GTID,
    ANONYMOUS_GTID,
    PREVIOUS_GTIDS,
    TRANSACTION_CONTEXT,
    VIEW_CHANGE,
    /**
     * Prepared XA transaction terminal event similar to XID except that it is specific to XA transaction.
     */
    XA_PREPARE;

    public static boolean isRowMutation(EventType eventType) {
        return EventType.isWrite(eventType) ||
               EventType.isUpdate(eventType) ||
               EventType.isDelete(eventType);
    }

    public static boolean isWrite(EventType eventType) {
        return eventType == PRE_GA_WRITE_ROWS ||
               eventType == WRITE_ROWS ||
               eventType == EXT_WRITE_ROWS;
    }

    public static boolean isUpdate(EventType eventType) {
        return eventType == PRE_GA_UPDATE_ROWS ||
               eventType == UPDATE_ROWS ||
               eventType == EXT_UPDATE_ROWS;
    }

    public static boolean isDelete(EventType eventType) {
        return eventType == PRE_GA_DELETE_ROWS ||
               eventType == DELETE_ROWS ||
               eventType == EXT_DELETE_ROWS;
    }

}

2.1 DML语句的事件

DML操作是指对数据库中表记录的操作,主要包括表记录的插入(insert)、更新(update)、删除(delete)和查询(select),是开发人员日常使用最频繁的操作。

监听的事件类型:ANONYMOUS_GTID
监听的事件类型:QUERY
监听的事件类型:TABLE_MAP
监听的事件类型:EXT_UPDATE_ROWS
监听的事件类型:XID

2.2 DDL语句事件

数据库模式定义语言DDL(Data Definition Language),是用于描述数据库中要存储的现实世界实体的语言。主要由create(添加)、alter(修改)、drop(删除)和 truncate(删除) 四个关键字完成。

例如:alter table test_table1 add column school_name varchar(255) not null;

监听的事件类型:ANONYMOUS_GTID
监听的事件类型:QUERY

2.3 端点续传事件

当binlog服务上线之后,服务就会在log文件的最新position处进行监听。格式如下:

Connected to localhost:3306 at mysql-bin.000013/1075 (sid:65535, cid:7)

但由于binlog服务下线,重新启动后,默认又开始在最新position处进行监听。会丢失一部门binlog的事件。故,每次事件均需要记录当前的position位置。重新建立client端时,使用记录的position位置。

故需要监听ROTATE事件

//当mysqld切换到新的二进制日志文件时写入。1. 当有人发出一个FLUSH LOGS 语句。或者当前二进制日志文件变大,超过max_binlog_size。
ROTATE,

2.4 不计入position更新(断点续传)的事件

  1. FORMAT_DESCRIPTION类型为binlog起始时间。
  2. HEARTBEAT为心跳检测事件,不会写入master的binlog,记录该事件的position会导致重启时报错。

3. 高可用&&端点续传

3.1 对于高可用

mysql-binlog-connector监听只能是单点服务。但是需要保证高可用,即某台机器挂掉,但是不能影响服务。

实现思路:部署多台机器,只有一台机器能监听binlog,当这台机器挂掉后,其他机器抢夺binlog的监听。

可以借助Zookeeper,也可以借助Redis实现。

3.2 对于端点续传。

mysql-binlog-connector没有帮我们实现断点续传,所以需要我们自己记录每个事件的position的位置。当重启服务时,自动读取记录的position位置进行识别。

import lombok.Data;


@Data
public class SyncConfig {
    /**
     * mysql的host配置
     */
    String host;
    /**
     * mysql的port配置
     */
    Integer port;
    /**
     * mysql的userName配置
     */
    String userName;
    /**
     * mysql的password配置
     */
    String password;
}
/**
 * 处理binlog位点信息接口,实现该接口创建自定义位点处理类
 */
public interface IPositionHandler {
    BinlogPositionEntity getPosition(SyncConfig syncConfig);

    void savePosition(SyncConfig syncConfig, BinlogPositionEntity binlogPositionEntity);
}
/**
 * 默认的端点续传存储器(需要借助分布式存储——此处可以保存到Redis中)
 */
public class DefaultPositionHandler implements IPositionHandler {

    private Map cache = new ConcurrentHashMap<>();

    @Override
    public BinlogPositionEntity getPosition(SyncConfig syncConfig) {
        return JSON.parseObject(cache.get(generateKey(syncConfig)), BinlogPositionEntity.class);
    }

    @Override
    public void savePosition(SyncConfig syncConfig, BinlogPositionEntity binlogPositionEntity) {
        cache.put(generateKey(syncConfig), JSON.toJSONString(binlogPositionEntity));
    }


    /**
     * @param syncConfig 参数配置
     * @return 生成的key
     */
    private String generateKey(SyncConfig syncConfig) {
        return syncConfig.getHost() + ":" + syncConfig.getPort();
    }

}
/**
 * 记录当前监听位置的事件
 *
 */
@Data
public class BinlogPositionEntity {
    /**
     * binlog文件的名字
     */
    private String binlogName;
    /**
     * binlog文件的位置
     */
    private Long position;
    /**
     * binlog的服务id
     */
    private Long serverId;
}

案例demo:

@Slf4j
public class TestBinlog {
    private static final SyncConfig syncConfig;

    private static IPositionHandler positionHandler = new DefaultPositionHandler();

    /**
     * 项目启动读取的配置
     */
    static {
        syncConfig = new SyncConfig();

        syncConfig.setHost("localhost");
        syncConfig.setPort(3306);
        syncConfig.setUserName("root");
        syncConfig.setPassword("123qwe");
    }

    public static void main(String[] args) throws IOException {

        BinaryLogClient client = new BinaryLogClient(syncConfig.getHost(), syncConfig.getPort(), syncConfig.getUserName(), syncConfig.getPassword());
        EventDeserializer eventDeserializer = new EventDeserializer();
        //时间反序列化的格式
//        eventDeserializer.setCompatibilityMode(
//                EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG,
//                EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY
//        );
        client.setEventDeserializer(eventDeserializer);
        //设置serverId,不同的集群,机器的serverId不能相同。
        client.setServerId(getRandomServerId());
        //获取position的位置(创建client时,读取当前记录的postion)
        BinlogPositionEntity binlogPositionEntity = positionHandler.getPosition(syncConfig);
        if (binlogPositionEntity != null &&
                binlogPositionEntity.getBinlogName() != null &&
                binlogPositionEntity.getPosition() != null) {
            client.setBinlogFilename(binlogPositionEntity.getBinlogName());
            client.setBinlogPosition(binlogPositionEntity.getPosition());
        }

        client.registerEventListener(new BinaryLogClient.EventListener() {

            @Override
            public void onEvent(Event event) {

                EventHeader header = event.getHeader();

                EventType eventType = header.getEventType();
                System.out.println("监听的事件类型:" + eventType);

                /*
                 * 不计入position更新的事件类型
                 * FORMAT_DESCRIPTION类型为binlog起始时间
                 * HEARTBEAT为心跳检测事件,不会写入master的binlog,记录该事件的position会导致重启时报错
                 */
                List excludePositionEventType = new ArrayList<>();
                excludePositionEventType.add(EventType.FORMAT_DESCRIPTION);
                excludePositionEventType.add(EventType.HEARTBEAT);
                if (!excludePositionEventType.contains(eventType)) {
                    BinlogPositionEntity binlogPositionEntity = new BinlogPositionEntity();
                    //处理rotate事件,这里会替换调binlog fileName
                    if (event.getHeader().getEventType().equals(EventType.ROTATE)) {
                        RotateEventData rotateEventData = (RotateEventData) event.getData();
                        binlogPositionEntity.setBinlogName(rotateEventData.getBinlogFilename());
                        binlogPositionEntity.setPosition(rotateEventData.getBinlogPosition());
                        binlogPositionEntity.setServerId(event.getHeader().getServerId());
                    } else { //统一处理事件对应的binlog position
                        //在Redis中获取获取binlog的position配置
                        binlogPositionEntity = positionHandler.getPosition(syncConfig);
                        EventHeaderV4 eventHeaderV4 = (EventHeaderV4) event.getHeader();
                        binlogPositionEntity.setPosition(eventHeaderV4.getPosition());
                        binlogPositionEntity.setServerId(event.getHeader().getServerId());
                    }
                    //将最新的配置保存到Redis中
                    log.info("保存的数据{}", JSON.toJSONString(binlogPositionEntity));
                    positionHandler.savePosition(syncConfig, binlogPositionEntity);

                    //todo 解析结果

                }


            }
        });
        client.connect();
    }

    private static long getRandomServerId() {
        try {
            return SecureRandom.getInstanceStrong().nextLong();
        } catch (NoSuchAlgorithmException e) {
            return RandomUtils.nextLong();
        }
    }

}

推荐的的binlog监听工具:
https://github.com/dothetrick/binlogportal

你可能感兴趣的:(JAVA实现mysql的binlog监听(mysql-binlog-connector))