MySQL二进制日志分析-QUERY_EVENT(包含代码实现)

QUERY_EVENT用于记录MySQL服务器中执行的SQL语句，早期版本无论是DDL还是DML的语句，都是记录在QUERY_EVENT中，如前文所说，由于基于语句的复制( statement-based replication - SBR)有很多问题，后续版本引入了ROWS_EVENT，DML语句可以基于值(行)复制(row-based replication - RBR)。基于语句的优势在于数据量小，需要更少的磁盘及网络的I/O，因为记录的是语句，可以用于审计。缺点在于一些语句的复制是不安全的，而且相较于RBR，select和inert语句需要跟多的行锁。

先看QUERY_EVENT的官方定义： The query event is used to send text querys right the binlog.

Post-header
  4              slave_proxy_id
  4              execution time
  1              schema length
  2              error-code
  if binlog-version ≥ 4:
    2              status-vars length
Payload
  string[$len]   status-vars
  string[$len]   schema
  1              [00]
  string[EOF]    query

QUERY_EVENT事件比较简单，通过Post-header的"schema length"和"status-vars length"获得Payload的"status-var"和"schema"的长度(字节数)。"query"部分一直到事件的结束，这里需要去掉checksum部分。根据上例改了下，附在本文结尾。

mysql> set @@global.binlog_format='STATEMENT';
Query OK, 0 rows affected (0.00 sec)

mysql> select @@binlog_format;
+-----------------+
| @@binlog_format |
+-----------------+
| STATEMENT       |
+-----------------+
1 row in set (0.00 sec)

mysql> create table my_test1(col1 timestamp, col2 varchar(20));
Query OK, 0 rows affected (0.26 sec)

mysql> insert into my_test1 values (sysdate(), user());
Query OK, 1 row affected, 1 warning (0.00 sec)

mysql> show warnings;
+-------+------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Level | Code | Message                                                                                                                                                                                                  |
+-------+------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Note  | 1592 | Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. |
+-------+------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)

程序解析的结果:

Mon Sep 30 09:52:49 CST 2019, start-pos: 1834199, end-pos: 1834330, thread-id: 274, execution-time: 0, error-code: 0, schema: tpcc
create table my_test1(col1 timestamp, col2 varchar(20))

Mon Sep 30 09:52:54 CST 2019, start-pos: 1834395, end-pos: 1834482, thread-id: 274, execution-time: 0, error-code: 0, schema: tpcc
BEGIN

Mon Sep 30 09:52:54 CST 2019, start-pos: 1834482, end-pos: 1834611, thread-id: 274, execution-time: 0, error-code: 0, schema: tpcc
insert into my_test1 values (sysdate(), user())

附上案例代码：

import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.util.Date;

import org.littlestar.mysql.binlog.parser.ParserHelper;

public class T2 {
    public static void main(String[] args) throws IOException {
        String binlogFileName = "D:\\build\\binlogs\\5.7.18\\blog.000020";
        RandomAccessFile binlogFile = new RandomAccessFile(binlogFileName, "r");
        FileChannel binlogFileChannel = binlogFile.getChannel();
        MappedByteBuffer blogFileBuffer = binlogFileChannel.map(MapMode.READ_ONLY, 0, binlogFile.length());
        ByteOrder order = ByteOrder.LITTLE_ENDIAN;
        // skip 4 bytes image number;
        blogFileBuffer.position(4);
        
        while (blogFileBuffer.remaining() > 19) {
            // Common Header..
            int startPos = blogFileBuffer.position();
            byte[] rawTimestamp = new byte[4];
            blogFileBuffer.get(rawTimestamp);
            
            long timestampValue = ParserHelper.getUnsignedLong(rawTimestamp, order);
            Date timestamp = new Date(timestampValue * 1000L);

            byte[] rawEventType = new byte[1];
            blogFileBuffer.get(rawEventType);
            int eventTypeValue = ParserHelper.getUnsignedInteger(rawEventType, order);

            byte[] rawServerId = new byte[4];
            blogFileBuffer.get(rawServerId);
            long serverId = ParserHelper.getUnsignedInteger(rawServerId, order);

            byte[] rawEventSize = new byte[4];
            blogFileBuffer.get(rawEventSize);
            long eventSize = ParserHelper.getUnsignedInteger(rawEventSize, order);

            byte[] rawLogPos = new byte[4];
            blogFileBuffer.get(rawLogPos);
            int logPos = ParserHelper.getUnsignedInteger(rawLogPos, order);
            
            byte[] rawFlags = new byte[2];
            blogFileBuffer.get(rawFlags);
            int flags = ParserHelper.getUnsignedInteger(rawFlags, order);
            
            if(eventTypeValue == 0x02) { //Query Event
                
                //post-header
                byte[] rawSlaveProxyId = new byte[4];
                byte[] rawExecutionTime = new byte[4];
                byte[] rawSchemaLength = new byte[1];
                byte[] rawErrorCode = new byte[2];

                blogFileBuffer.get(rawSlaveProxyId);
                blogFileBuffer.get(rawExecutionTime);
                blogFileBuffer.get(rawSchemaLength);
                blogFileBuffer.get(rawErrorCode);
                long slaveProxyId = ParserHelper.getUnsignedInteger(rawSlaveProxyId, order);
                long executionTime = ParserHelper.getUnsignedInteger(rawExecutionTime, order);
                int schemaLength = ParserHelper.getUnsignedInteger(rawSchemaLength, order);
                int errorCode = ParserHelper.getUnsignedInteger(rawErrorCode, order);
                
                //if Binlog Version >= 4
                byte[] rawStatusVarsLength = new byte[2];
                blogFileBuffer.get(rawStatusVarsLength);
                int statusVarsLength = ParserHelper.getUnsignedInteger(rawStatusVarsLength, order);
                
                //Payload
                if (statusVarsLength > 0) {
                    byte[] rawStatusVars = new byte[statusVarsLength];
                    blogFileBuffer.get(rawStatusVars);
                }
                byte[] rawSchema = new byte[schemaLength + 1]; // with 1 byte null(00);
                blogFileBuffer.get(rawSchema);
                String schema = new String(rawSchema).trim();
                
                byte[] remainBytes = new byte[logPos - blogFileBuffer.position()];
                blogFileBuffer.get(remainBytes);
                byte[] rawQuery = ParserHelper.getNulTerminatedString(remainBytes, 0, (remainBytes.length - 4)); // CRC32 4 bytes.
                String query = new String(rawQuery).trim();
                
                System.out.println(timestamp.toString()
                        + ", start-pos: " + startPos
                        + ", end-pos: " + logPos
                        + ", thread-id: " + slaveProxyId
                        + ", execution-time: " + executionTime 
                        + ", error-code: " + errorCode
                        + ", schema: " + schema
                        + "\n" + query + "\n"
                        );
            }
            blogFileBuffer.position(logPos); //next event position
        }
        binlogFileChannel.close();
        binlogFile.close();
        
    }
}