MySQL 8.0文档阅读: binlog文件格式

前言

官方文档
https://dev.mysql.com/doc/internals/en/replication-protocol.html
但是文档似乎没有写全, 有些事件的定义找不到文档

binlog文件分析

以下是某个binlog文件的分析结果 (貌似都是小端字节序)

FE 62 69 6E           // magic FE 'bin'

35 37 8F 5C           // 事件创建的时间, 秒, 0x5C8F3735, 即2019/3/18 14:14:13
0F                    // event type, 15表示binlog描述事件
01 00 00 00           // server id, 01
78 00 00 00           // event size, 事件大小, 120字节. 包括header, post-header, body
7C 00 00 00           // 下一个事件的位置, 
01 00                 // flag, 即0x0001. LOG_EVENT_BINLOG_IN_USE_F, 
                      // 如果当前文件正在被mysql使用, 则置成1, 否则, 置为0
04 00                 // binlog-version, 0x0004, 
38 2E 30 2E 31 35 00  // mysql-server版本, 50个字节, 本处为字符串"8.0.15"
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
00 00 00 00 00 00 00 00 00 00 00 
35 37 8F 5C           // binlog创建时间
13                    // Binlog Event Header的消息头大小, 固定值19
00 0D 00 08 00 00 00 00 04 00 04 00 00 00 60 00  // 以下三行是为各种类型的Header所对应的长度大小
04 1A 08 00 00 00 08 08 08 02 00 00 00 0A 0A 0A  // 源代码中有40种type定义 
2A 2A 00 12 34 00 0A 
01 F7 B8 86 9B  // 这一段不知道是什么东西, 可能是检验和之类的东西

35 37 8F 5C // 时间
23 //事件类型35, 表示PREVIOUS_GTIDS_LOG_EVENT
01 00 00 00 
47 00 00 00       // 事件长度71字节
C3 00 00 00       // 下一个事件的位置
80 00             // flag 
01 00 00 00 00 00 00 00 6F 16 6D 02 44 84 11 E9  //这段数据我也不知道怎么解析, 文档上没有. 估计得看源代码了
8A 8E 00 16 3E 10 05 86 01 00 00 00 00 00 00 00 
01 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 BA
09 A9 48 

AA 37 8F 5C  // 时间
21  //事件类型33, 表示GTID_LOG_EVENT
01 00 00 00 
4F 00 00 00
12 01 00 00 
00 00 
00 6F 16 6D 02 44 84 11 E9 8A
8E 00 16 3E 10 05 86 09 00 00 00 00 00 00 00 02
00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00
15 07 24 53 58 84 05 FC 11 01 8F 38 01 00 20 9E
37 7F 

AA 37 8F 5C 
02    //QUERY_EVENT
01 00 00 00 
4B 00 00 00 
5D 01 00 00 
08 00 
0B 00 00 00  // slave_proxy_id
00 00 00 00  // execution time
04           // schema length
00 00        // error-code
1D 00        //status-vars length, 29,  官网上似乎没写完整
00 00 00 00 00 01 20 00 A0 45 00 00 00 00
06 03 73 74 64 04 FF 00 FF 00 FF 00 12 FF 00 

74 65 73 74 00  //schema, "test"
42 45 47 49 4E  4B 0E 6C  //字符串 BEGIN K l,  不清楚这是什么语句.
A5 //估计是校验和

AA 37 8F 5C 
13   //TABLE_MAP_EVENT, 官网上没写. 不知道这该如何定义
01 00 00 00 
30 00 00 00 
8D 01 00 00 
00 00
45 00 00 00 00 00 01 00 04 74 65 73 74 00 02 74
31 00 01 03 00 01 01 01 00 DD 9F 06 84 AA 37 8F
5C 1E 01 00 00 00 28 00 00 00 B5 01 00 00 00 00
45 00 00 00 00 00 01 00 02 00 01 FF 00 09 00 00
00 A0 D7 5B 35 

AA 37 8F 5C 
10    // XID_EVENT 
01 00 00 00 
1F 00 00 00 
D4 01 00 00 
00 00 
17 00 00 00 00 00 00 00
7F 0D 0C AF                                    

mysqlbinlog主程序

mysql的mysqlbinlog可以使用binlog文件, 查看它的源码. 通过main函数一路追踪, 它处理本机的binlog文件时, 使用的以下代码
mysqlbinlog.cc

static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
                                          const char *logname) {
  Exit_status retval = OK_CONTINUE;

  ulong max_event_size = 0;
  mysql_get_option(NULL, MYSQL_OPT_MAX_ALLOWED_PACKET, &max_event_size);
  
  Mysqlbinlog_file_reader mysqlbinlog_file_reader(opt_verify_binlog_checksum,
                                                  max_event_size);
  Format_description_log_event *fdle = nullptr;
  //打开binlog文件, 它会检查文件头的4字节magic FE 62 69 6E
  //start_position默认4 , 即跳过magic
  if (mysqlbinlog_file_reader.open(logname, start_position, &fdle)) {
    error("%s", mysqlbinlog_file_reader.get_error_str());
    return ERROR_STOP;
  }

  //处理binlog描述事件. binlog的第一个事件是描述
  //如果start_position不是文件开始位置, 那么就需要读取fdle
  //如果start_position未设置, 应当不会执行这段
  if (fdle != nullptr) {
    retval = process_event(print_event_info, fdle,
                           mysqlbinlog_file_reader.event_start_pos(), logname);
    if (retval != OK_CONTINUE) goto end;
  }

  //不清楚作用, 跳过
  if (strcmp(logname, "-") == 0)
    mysqlbinlog_file_reader.event_data_istream()->set_multi_binlog_magic();

 //主循环, 不断读取和处理event, 直到stop_position
  for (;;) {
    char llbuff[21];
    my_off_t old_off = mysqlbinlog_file_reader.position();

    Log_event *ev = mysqlbinlog_file_reader.read_event_object();
    if (ev == NULL) {
      /*
        if binlog wasn't closed properly ("in use" flag is set) don't complain
        about a corruption, but treat it as EOF and move to the next binlog.
      */
      if ((mysqlbinlog_file_reader.format_description_event()->header()->flags &
           LOG_EVENT_BINLOG_IN_USE_F) ||
          mysqlbinlog_file_reader.get_error_type() ==
              Binlog_read_error::READ_EOF)
        goto end;

      error(
          "Could not read entry at offset %s: "
          "Error in log format or read error 1.",
          llstr(old_off, llbuff));
      error("%s", mysqlbinlog_file_reader.get_error_str());
      goto err;
    }
    if ((retval = process_event(print_event_info, ev, old_off, logname)) !=
        OK_CONTINUE)
      goto end;
  }

  /* NOTREACHED */

err:
  retval = ERROR_STOP;

end:
  return retval;
}

Mysqlbinlog_file_reader

typedef Basic_binlog_file_reader<
    Mysqlbinlog_ifile, Mysqlbinlog_event_data_istream,
    Binlog_event_object_istream, Default_binlog_event_allocator>
    Mysqlbinlog_file_reader;

Mysqlbinlog_ifile是对文件的再次封装(open, read, close等), 它会检查并跳过文件的magic

Mysqlbinlog_event_data_istream继承了Binlog_event_data_istream, 用于读取event的消息头和消息体
位于binlog_reader.h

Binlog_event_object_istream将event的二进制数据反序列化成Log_event对象

Log_event_type

binlog_event.h

enum Log_event_type {
  /**
    Every time you add a type, you have to
    - Assign it a number explicitly. Otherwise it will cause trouble
      if a event type before is deprecated and removed directly from
      the enum.
    - Fix Format_description_event::Format_description_event().
  */
  UNKNOWN_EVENT = 0,
  /*
    Deprecated since mysql 8.0.2. It is just a placeholder,
    should not be used anywhere else.
  */
  START_EVENT_V3 = 1,
  QUERY_EVENT = 2,
  STOP_EVENT = 3,
  ROTATE_EVENT = 4,
  INTVAR_EVENT = 5,

  SLAVE_EVENT = 7,

  APPEND_BLOCK_EVENT = 9,
  DELETE_FILE_EVENT = 11,

  RAND_EVENT = 13,
  USER_VAR_EVENT = 14,
  FORMAT_DESCRIPTION_EVENT = 15,
  XID_EVENT = 16,
  BEGIN_LOAD_QUERY_EVENT = 17,
  EXECUTE_LOAD_QUERY_EVENT = 18,

  TABLE_MAP_EVENT = 19,

  /**
    The V1 event numbers are used from 5.1.16 until mysql-5.6.
  */
  WRITE_ROWS_EVENT_V1 = 23,
  UPDATE_ROWS_EVENT_V1 = 24,
  DELETE_ROWS_EVENT_V1 = 25,

  /**
    Something out of the ordinary happened on the master
   */
  INCIDENT_EVENT = 26,

  /**
    Heartbeat event to be send by master at its idle time
    to ensure master's online status to slave
  */
  HEARTBEAT_LOG_EVENT = 27,

  /**
    In some situations, it is necessary to send over ignorable
    data to the slave: data that a slave can handle in case there
    is code for handling it, but which can be ignored if it is not
    recognized.
  */
  IGNORABLE_LOG_EVENT = 28,
  ROWS_QUERY_LOG_EVENT = 29,

  /** Version 2 of the Row events */
  WRITE_ROWS_EVENT = 30,
  UPDATE_ROWS_EVENT = 31,
  DELETE_ROWS_EVENT = 32,

  GTID_LOG_EVENT = 33,
  ANONYMOUS_GTID_LOG_EVENT = 34,

  PREVIOUS_GTIDS_LOG_EVENT = 35,

  TRANSACTION_CONTEXT_EVENT = 36,

  VIEW_CHANGE_EVENT = 37,

  /* Prepared XA transaction terminal event similar to Xid */
  XA_PREPARE_LOG_EVENT = 38,

  /**
    Extension of UPDATE_ROWS_EVENT, allowing partial values according
    to binlog_row_value_options.
  */
  PARTIAL_UPDATE_ROWS_EVENT = 39,

  /**
    Add new events here - right above this comment!
    Existing events (except ENUM_END_EVENT) should never change their numbers
  */
  ENUM_END_EVENT /* end marker */
};

宏

binlog_event.h:

//binlog事件头的最小大小
#define LOG_EVENT_MINIMAL_HEADER_LEN 19U


#define EVENT_TYPE_OFFSET 4
#define SERVER_ID_OFFSET 5
#define EVENT_LEN_OFFSET 9
#define LOG_POS_OFFSET 13
#define FLAGS_OFFSET 17

/** start event post-header (for v3 and v4) */
#define ST_BINLOG_VER_OFFSET 0
#define ST_SERVER_VER_OFFSET 2
#define ST_CREATED_OFFSET (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN)
#define ST_COMMON_HEADER_LEN_OFFSET (ST_CREATED_OFFSET + 4)

#define LOG_EVENT_HEADER_LEN 19U /* the fixed header length */

log_event.h

//binlog文件的magic
#define BINLOG_MAGIC "\xfe\x62\x69\x6e"
#define BINLOG_MAGIC_SIZE 4