canal 源码解析（2）-数据流转篇（3）
event-————》entry
一、准备
先确认下当前位点信息，最新位点信息是1601，
执行命令：show BINLOG EVENTS in "mysql-bin.000031";
而项目目前位点信息是761 ，和1601 差别太多，是因为中间多了，插入，更新，删除操作。如下
二、解析插入，更新，删除操作事件。
2.1 ANONYMOUS_GTID_LOG_EVENT=34 该事件是mysql5.7 以后自动开启的，表示事物id，每个事物开始之前都会存在一个该事件，注释：
event： 
因为header.type=34 ，sink 目前直接返回null，
2.2、QUERY_EVENT=2 该event 以文本记录binlog的操作
根据type=2
代码如下：querystring=begin，然后构造transactionbegin对象，
private Entry parseQueryEvent(QueryLogEvent event, boolean isSeek) { String queryString = event.getQuery(); if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) { TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId()); Header header = createHeader(binlogFileName, event.getHeader(), "", "", null); return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString()); } else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) { TransactionEnd transactionEnd = createTransactionEnd(0L); // MyISAM可能不会有xid事件 Header header = createHeader(binlogFileName, event.getHeader(), "", "", null); return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString()); } else { boolean notFilter = false; EventType type = EventType.QUERY; String tableName = null; String schemaName = null; if (useDruidDdlFilter) { Listresults = DruidDdlParser.parse(queryString, event.getDbName()); for (DdlResult result : results) { if (!processFilter(queryString, result)) { // 只要有一个数据不进行过滤 notFilter = true; } } if (results.size() > 0) { // 如果针对多行的DDL,只能取第一条 type = results.get(0).getType(); schemaName = results.get(0).getSchemaName(); tableName = results.get(0).getTableName(); } } else { DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName()); if (!processFilter(queryString, result)) { notFilter = true; } type = result.getType(); schemaName = result.getSchemaName(); tableName = result.getTableName(); } if (!notFilter) { // 如果是过滤的数据就不处理了 return null; } if (!isSeek) { // 使用新的表结构元数据管理方式 EntryPosition position = createPosition(event.getHeader()); tableMetaCache.apply(position, event.getDbName(), queryString, null); } Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); if (type != EventType.QUERY) { rowChangeBuider.setIsDdl(true); } rowChangeBuider.setSql(queryString); if (StringUtils.isNotEmpty(event.getDbName())) {// 可能为空 rowChangeBuider.setDdlSchemaName(event.getDbName()); } rowChangeBuider.setEventType(type); return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString()); } }
构造header对象（使用probuf）
private Header createHeader(String binlogFile, LogHeader logHeader, String schemaName, String tableName, EventType eventType) { // header会做信息冗余,方便以后做检索或者过滤 Header.Builder headerBuilder = Header.newBuilder(); headerBuilder.setVersion(version); headerBuilder.setLogfileName(binlogFile); headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen()); headerBuilder.setServerId(logHeader.getServerId()); headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode headerBuilder.setExecuteTime(logHeader.getWhen() * 1000L); headerBuilder.setSourceType(Type.MYSQL); if (eventType != null) { headerBuilder.setEventType(eventType); } if (schemaName != null) { headerBuilder.setSchemaName(schemaName); } if (tableName != null) { headerBuilder.setTableName(tableName); } headerBuilder.setEventLength(logHeader.getEventLen()); // enable gtid position if (gtidSet != null) { String gtid = gtidSet.toString(); headerBuilder.setGtid(gtid); } return headerBuilder.build(); }构造entry对象（使用probuf）
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) { Entry.Builder entryBuilder = Entry.newBuilder(); entryBuilder.setHeader(header); entryBuilder.setEntryType(entryType); entryBuilder.setStoreValue(storeValue); return entryBuilder.build(); }
最终对象为：
BEGIN std test
header {
version: 1logfileName: "mysql-bin.000031"
logfileOffset: 460
serverId: 1
serverenCode: "UTF-8"
executeTime: 1529994671000
sourceType: MYSQL
schemaName: ""
tableName: ""
eventLength: 72
}
entryType: TRANSACTIONBEGIN
storeValue: " \200\001"
2.3、 TABLE_MAP_EVENT=19 数据库有5列，
sink过滤： 跳过
2.4、WRITE_ROWS_EVENT=30 插入的对象在rowsbuf（byte）里
插入，删除，更新，在sink模块里共用的是同一个 event方法，如下
代码
private Entry parseRowsEvent(RowsLogEvent event) { if (filterRows) { return null; } try { TableMapLogEvent table = event.getTable(); if (table == null) { // tableId对应的记录不存在 throw new TableIdNotFoundException("not found tableId:" + event.getTableId()); } boolean isHeartBeat = isAliSQLHeartBeat(table.getDbName(), table.getTableName()); boolean isRDSHeartBeat = tableMetaCache.isOnRDS() && isRDSHeartBeat(table.getDbName(), table.getTableName()); String fullname = table.getDbName() + "." + table.getTableName(); // check name filter if (nameFilter != null && !nameFilter.filter(fullname)) { return null; } if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) { return null; } // if (isHeartBeat || isRDSHeartBeat) { // // 忽略rds模式的mysql.ha_health_check心跳数据 // return null; // } TableMeta tableMeta = null; if (isRDSHeartBeat) { // 处理rds模式的mysql.ha_health_check心跳数据 // 主要RDS的心跳表基本无权限,需要mock一个tableMeta FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0"); FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0"); tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta)); } else if (isHeartBeat) { // 处理alisql模式的test.heartbeat心跳数据 // 心跳表基本无权限,需要mock一个tableMeta FieldMeta idMeta = new FieldMeta("id", "smallint(6)", false, true, null); FieldMeta typeMeta = new FieldMeta("type", "int(11)", true, false, null); tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta)); } EventType eventType = null; int type = event.getHeader().getType(); if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) { eventType = EventType.INSERT; } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) { eventType = EventType.UPDATE; } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) { eventType = EventType.DELETE; } else { throw new CanalParseException("unsupport event type :" + event.getHeader().getType()); } Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType); EntryPosition position = createPosition(event.getHeader()); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); rowChangeBuider.setTableId(event.getTableId()); rowChangeBuider.setIsDdl(false); rowChangeBuider.setEventType(eventType); RowsLogBuffer buffer = event.getRowsBuf(charset.name()); BitSet columns = event.getColumns(); BitSet changeColumns = event.getChangeColumns(); boolean tableError = false; if (tableMetaCache != null && tableMeta == null) {// 入错存在table meta // cache tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true, position); if (tableMeta == null) { tableError = true; if (!filterTableError) { throw new CanalParseException("not found [" + fullname + "] in db , pls check!"); } } } while (buffer.nextOneRow(columns)) { // 处理row记录 RowData.Builder rowDataBuilder = RowData.newBuilder(); if (EventType.INSERT == eventType) { // insert的记录放在before字段中 tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta); } else if (EventType.DELETE == eventType) { // delete的记录放在before字段中 tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta); } else { // update需要处理before/after tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta); if (!buffer.nextOneRow(changeColumns)) { rowChangeBuider.addRowDatas(rowDataBuilder.build()); break; } tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta); } rowChangeBuider.addRowDatas(rowDataBuilder.build()); } RowChange rowChange = rowChangeBuider.build(); if (tableError) { Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY); logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString()); return null; } else { Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString()); return entry; } } catch (Exception e) { throw new CanalParseException("parse row data failed.", e); } }
解析思路：
1、获取table。
2、心跳，rds判断
3、获取库.表 test.student
4、对eventype归纳
5、组装header（如上代码）
（ version: 1
logfileName: "mysql-bin.000031"logfileOffset: 894
serverId: 1
serverenCode: "UTF-8"
executeTime: 1529994880000
sourceType: MYSQL
schemaName: "test"
tableName: "student"
eventLength: 60
eventType: INSERT）
6、组装position
EntryPosition[included=false,journalName=mysql-bin.000031,position=954,serverId=1,gtid=
7、从db2里获取tableMeta（表结构信息）
public TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position) { TableMeta tableMeta = null; if (tableMetaTSDB != null) { tableMeta = tableMetaTSDB.find(schema, table); if (tableMeta == null) { // 因为条件变化，可能第一次的tableMeta没取到，需要从db获取一次，并记录到snapshot中 String fullName = getFullName(schema, table); try { ResultSetPacket packet = connection.query("show create table " + fullName); String createDDL = null; if (packet.getFieldValues().size() > 0) { createDDL = packet.getFieldValues().get(1); } // 强制覆盖掉内存值 tableMetaTSDB.apply(position, schema, createDDL, "first"); tableMeta = tableMetaTSDB.find(schema, table); } catch (IOException e) { throw new CanalParseException("fetch failed by table meta:" + fullName, e); } } return tableMeta; } else { if (!useCache) { tableMetaDB.invalidate(getFullName(schema, table)); } return tableMetaDB.getUnchecked(getFullName(schema, table)); } }值：TableMeta [schema=test, table=student, fileds=
FieldMeta [columnName=id, columnType=int(11), nullable=false, key=true, defaultValue=null, extra=null, unique=false]
FieldMeta [columnName=name, columnType=varchar(255), nullable=true, key=false, defaultValue=null, extra=null, unique=false]
FieldMeta [columnName=phone, columnType=varchar(255), nullable=true, key=false, defaultValue=null, extra=null, unique=false]
FieldMeta [columnName=mark, columnType=varchar(255), nullable=true, key=false, defaultValue=null, extra=null, unique=false]
FieldMeta [columnName=creat_time, columnType=datetime, nullable=true, key=false, defaultValue=null, extra=null, unique=false]
]
8.、更改nullbits，开始遍历tableMeta
9. 处理rows并归纳insert，update，delete
private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException { int columnCnt = event.getTable().getColumnCnt(); ColumnInfo[] columnInfo = event.getTable().getColumnInfo(); boolean tableError = false; // check table fileds count，只能处理加字段 boolean existRDSNoPrimaryKey = false; if (tableMeta != null && columnInfo.length > tableMeta.getFields().size()) { if (tableMetaCache.isOnRDS()) { // 特殊处理下RDS的场景 ListprimaryKeys = tableMeta.getPrimaryFields(); if (primaryKeys == null || primaryKeys.isEmpty()) { if (columnInfo.length == tableMeta.getFields().size() + 1 && columnInfo[columnInfo.length - 1].type == LogEvent.MYSQL_TYPE_LONGLONG) { existRDSNoPrimaryKey = true; } } } EntryPosition position = createPosition(event.getHeader()); if (!existRDSNoPrimaryKey) { // online ddl增加字段操作步骤： // 1. 新增一张临时表，将需要做ddl表的数据全量导入 // 2. 在老表上建立I/U/D的trigger，增量的将数据插入到临时表 // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作 // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作 // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作 tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false, position);// 强制重新获取一次 if (tableMeta == null) { tableError = true; if (!filterTableError) { throw new CanalParseException("not found [" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] in db , pls check!"); } } // 在做一次判断 if (tableMeta != null && columnInfo.length > tableMeta.getFields().size()) { tableError = true; if (!filterTableError) { throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() + "," + columnInfo.length + " vs " + tableMeta.getFields().size()); } } } else { logger.warn("[" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] is no primary key , skip alibaba_rds_row_id column"); } } for (int i = 0; i < columnCnt; i++) { ColumnInfo info = columnInfo[i]; // mysql 5.6开始支持nolob/mininal类型,并不一定记录所有的列,需要进行判断 if (!cols.get(i)) { continue; } if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) { // 不解析最后一列 buffer.nextValue(info.type, info.meta, false); continue; } Column.Builder columnBuilder = Column.newBuilder(); FieldMeta fieldMeta = null; if (tableMeta != null && !tableError) { // 处理file meta fieldMeta = tableMeta.getFields().get(i); columnBuilder.setName(fieldMeta.getColumnName()); columnBuilder.setIsKey(fieldMeta.isKey()); // 增加mysql type类型,issue 73 columnBuilder.setMysqlType(fieldMeta.getColumnType()); } columnBuilder.setIndex(i); columnBuilder.setIsNull(false); // fixed issue // https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理 boolean isBinary = false; if (fieldMeta != null) { if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) { isBinary = true; } else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "BINARY")) { isBinary = true; } } buffer.nextValue(info.type, info.meta, isBinary); if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) { // 不解析最后一列 continue; } int javaType = buffer.getJavaType(); if (buffer.isNull()) { columnBuilder.setIsNull(true); } else { final Serializable value = buffer.getValue(); // 处理各种类型 switch (javaType) { case Types.INTEGER: case Types.TINYINT: case Types.SMALLINT: case Types.BIGINT: // 处理unsigned类型 Number number = (Number) value; if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) { switch (buffer.getLength()) { case 1: /* MYSQL_TYPE_TINY */ columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE + number.intValue()))); javaType = Types.SMALLINT; // 往上加一个量级 break; case 2: /* MYSQL_TYPE_SHORT */ columnBuilder.setValue(String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE + number.intValue()))); javaType = Types.INTEGER; // 往上加一个量级 break; case 3: /* MYSQL_TYPE_INT24 */ columnBuilder.setValue(String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE + number.intValue()))); javaType = Types.INTEGER; // 往上加一个量级 break; case 4: /* MYSQL_TYPE_LONG */ columnBuilder.setValue(String.valueOf(Long.valueOf(INTEGER_MAX_VALUE + number.longValue()))); javaType = Types.BIGINT; // 往上加一个量级 break; case 8: /* MYSQL_TYPE_LONGLONG */ columnBuilder.setValue(BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue())) .toString()); javaType = Types.DECIMAL; // 往上加一个量级，避免执行出错 break; } } else { // 对象为number类型，直接valueof即可 columnBuilder.setValue(String.valueOf(value)); } break; case Types.REAL: // float case Types.DOUBLE: // double // 对象为number类型，直接valueof即可 columnBuilder.setValue(String.valueOf(value)); break; case Types.BIT:// bit // 对象为number类型 columnBuilder.setValue(String.valueOf(value)); break; case Types.DECIMAL: columnBuilder.setValue(((BigDecimal) value).toPlainString()); break; case Types.TIMESTAMP: // 修复时间边界值 // String v = value.toString(); // v = v.substring(0, v.length() - 2); // columnBuilder.setValue(v); // break; case Types.TIME: case Types.DATE: // 需要处理year columnBuilder.setValue(value.toString()); break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: // fixed text encoding // https://github.com/AlibabaTech/canal/issues/18 // mysql binlog中blob/text都处理为blob类型，需要反查table // meta，按编码解析text if (fieldMeta != null && isText(fieldMeta.getColumnType())) { columnBuilder.setValue(new String((byte[]) value, charset)); javaType = Types.CLOB; } else { // byte数组，直接使用iso-8859-1保留对应编码，浪费内存 columnBuilder.setValue(new String((byte[]) value, ISO_8859_1)); javaType = Types.BLOB; } break; case Types.CHAR: case Types.VARCHAR: columnBuilder.setValue(value.toString()); break; default: columnBuilder.setValue(value.toString()); } } columnBuilder.setSqlType(javaType); // 设置是否update的标记位 columnBuilder.setUpdated(isAfter && isUpdate(rowDataBuilder.getBeforeColumnsList(), columnBuilder.getIsNull() ? null : columnBuilder.getValue(), i)); if (isAfter) { rowDataBuilder.addAfterColumns(columnBuilder.build()); } else { rowDataBuilder.addBeforeColumns(columnBuilder.build()); } } return tableError; }
9.1 获取变更的数据
buffer.nextValue(info.type, info.meta, isBinary);
public final Serializable nextValue(final int type, final int meta, boolean isBinary) { fNull = nullBits.get(nullBitIndex++); if (fNull) { value = null; javaType = mysqlToJavaType(type, meta, isBinary); length = 0; return null; } else { // Extracting field value from packed buffer. return fetchValue(type, meta, isBinary); } }
9.2过滤rowsbuff规则如下
final Serializable fetchValue(int type, final int meta, boolean isBinary) { int len = 0; if (type == LogEvent.MYSQL_TYPE_STRING) { if (meta >= 256) { int byte0 = meta >> 8; int byte1 = meta & 0xff; if ((byte0 & 0x30) != 0x30) { /* a long CHAR() field: see #37426 */ len = byte1 | (((byte0 & 0x30) ^ 0x30) << 4); type = byte0 | 0x30; } else { switch (byte0) { case LogEvent.MYSQL_TYPE_SET: case LogEvent.MYSQL_TYPE_ENUM: case LogEvent.MYSQL_TYPE_STRING: type = byte0; len = byte1; break; default: throw new IllegalArgumentException(String.format("!! Don't know how to handle column type=%d meta=%d (%04X)", type, meta, meta)); } } } else { len = meta; } } switch (type) { case LogEvent.MYSQL_TYPE_LONG: { // XXX: How to check signed / unsigned? // value = unsigned ? Long.valueOf(buffer.getUint32()) : // Integer.valueOf(buffer.getInt32()); value = Integer.valueOf(buffer.getInt32()); javaType = Types.INTEGER; length = 4; break; } case LogEvent.MYSQL_TYPE_TINY: { // XXX: How to check signed / unsigned? // value = Integer.valueOf(unsigned ? buffer.getUint8() : // buffer.getInt8()); value = Integer.valueOf(buffer.getInt8()); javaType = Types.TINYINT; // java.sql.Types.INTEGER; length = 1; break; } case LogEvent.MYSQL_TYPE_SHORT: { // XXX: How to check signed / unsigned? // value = Integer.valueOf(unsigned ? buffer.getUint16() : // buffer.getInt16()); value = Integer.valueOf((short) buffer.getInt16()); javaType = Types.SMALLINT; // java.sql.Types.INTEGER; length = 2; break; } case LogEvent.MYSQL_TYPE_INT24: { // XXX: How to check signed / unsigned? // value = Integer.valueOf(unsigned ? buffer.getUint24() : // buffer.getInt24()); value = Integer.valueOf(buffer.getInt24()); javaType = Types.INTEGER; length = 3; break; } case LogEvent.MYSQL_TYPE_LONGLONG: { // XXX: How to check signed / unsigned? // value = unsigned ? buffer.getUlong64()) : // Long.valueOf(buffer.getLong64()); value = Long.valueOf(buffer.getLong64()); javaType = Types.BIGINT; // Types.INTEGER; length = 8; break; } case LogEvent.MYSQL_TYPE_DECIMAL: { /* * log_event.h : This enumeration value is only used internally * and cannot exist in a binlog. */ logger.warn("MYSQL_TYPE_DECIMAL : This enumeration value is " + "only used internally and cannot exist in a binlog!"); javaType = Types.DECIMAL; value = null; /* unknown format */ length = 0; break; } case LogEvent.MYSQL_TYPE_NEWDECIMAL: { final int precision = meta >> 8; final int decimals = meta & 0xff; value = buffer.getDecimal(precision, decimals); javaType = Types.DECIMAL; length = precision; break; } case LogEvent.MYSQL_TYPE_FLOAT: { value = Float.valueOf(buffer.getFloat32()); javaType = Types.REAL; // Types.FLOAT; length = 4; break; } case LogEvent.MYSQL_TYPE_DOUBLE: { value = Double.valueOf(buffer.getDouble64()); javaType = Types.DOUBLE; length = 8; break; } case LogEvent.MYSQL_TYPE_BIT: { /* Meta-data: bit_len, bytes_in_rec, 2 bytes */ final int nbits = ((meta >> 8) * 8) + (meta & 0xff); len = (nbits + 7) / 8; if (nbits > 1) { // byte[] bits = new byte[len]; // buffer.fillBytes(bits, 0, len); // 转化为unsign long switch (len) { case 1: value = buffer.getInt8(); break; case 2: value = buffer.getBeUint16(); break; case 3: value = buffer.getBeUint24(); break; case 4: value = buffer.getBeUint32(); break; case 5: value = buffer.getBeUlong40(); break; case 6: value = buffer.getBeUlong48(); break; case 7: value = buffer.getBeUlong56(); break; case 8: value = buffer.getBeUlong64(); break; default: throw new IllegalArgumentException("!! Unknown Bit len = " + len); } } else { final int bit = buffer.getInt8(); // value = (bit != 0) ? Boolean.TRUE : Boolean.FALSE; value = bit; } javaType = Types.BIT; length = nbits; break; } case LogEvent.MYSQL_TYPE_TIMESTAMP: { // MYSQL DataTypes: TIMESTAMP // range is '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' // UTC // A TIMESTAMP cannot represent the value '1970-01-01 00:00:00' // because that is equivalent to 0 seconds from the epoch and // the value 0 is reserved for representing '0000-00-00 // 00:00:00', the “zero” TIMESTAMP value. final long i32 = buffer.getUint32(); if (i32 == 0) { value = "0000-00-00 00:00:00"; } else { String v = new Timestamp(i32 * 1000).toString(); value = v.substring(0, v.length() - 2); } javaType = Types.TIMESTAMP; length = 4; break; } case LogEvent.MYSQL_TYPE_TIMESTAMP2: { final long tv_sec = buffer.getBeUint32(); // big-endian int tv_usec = 0; switch (meta) { case 0: tv_usec = 0; break; case 1: case 2: tv_usec = buffer.getInt8() * 10000; break; case 3: case 4: tv_usec = buffer.getBeInt16() * 100; break; case 5: case 6: tv_usec = buffer.getBeInt24(); break; default: tv_usec = 0; break; } String second = null; if (tv_sec == 0) { second = "0000-00-00 00:00:00"; } else { Timestamp time = new Timestamp(tv_sec * 1000); second = time.toString(); second = second.substring(0, second.length() - 2);// 去掉毫秒精度.0 } if (meta >= 1) { String microSecond = usecondsToStr(tv_usec, meta); microSecond = microSecond.substring(0, meta); value = second + '.' + microSecond; } else { value = second; } javaType = Types.TIMESTAMP; length = 4 + (meta + 1) / 2; break; } case LogEvent.MYSQL_TYPE_DATETIME: { // MYSQL DataTypes: DATETIME // range is '0000-01-01 00:00:00' to '9999-12-31 23:59:59' final long i64 = buffer.getLong64(); /* YYYYMMDDhhmmss */ if (i64 == 0) { value = "0000-00-00 00:00:00"; } else { final int d = (int) (i64 / 1000000); final int t = (int) (i64 % 1000000); // if (cal == null) cal = Calendar.getInstance(); // cal.clear(); /* month is 0-based, 0 for january. */ // cal.set(d / 10000, (d % 10000) / 100 - 1, d % 100, t / // 10000, (t % 10000) / 100, t % 100); // value = new Timestamp(cal.getTimeInMillis()); // value = String.format("%04d-%02d-%02d %02d:%02d:%02d", // d / 10000, // (d % 10000) / 100, // d % 100, // t / 10000, // (t % 10000) / 100, // t % 100); StringBuilder builder = new StringBuilder(); builder.append(formatNumber(d / 10000, 4)) .append('-') .append(formatNumber((d % 10000) / 100, 2)) .append('-') .append(formatNumber(d % 100, 2)) .append(' ') .append(formatNumber(t / 10000, 2)) .append(':') .append(formatNumber((t % 10000) / 100, 2)) .append(':') .append(formatNumber(t % 100, 2)); value = builder.toString(); } javaType = Types.TIMESTAMP; length = 8; break; } case LogEvent.MYSQL_TYPE_DATETIME2: { /* * DATETIME and DATE low-level memory and disk representation * routines 1 bit sign (used when on disk) 17 bits year*13+month * (year 0-9999, month 0-12) 5 bits day (0-31) 5 bits hour * (0-23) 6 bits minute (0-59) 6 bits second (0-59) 24 bits * microseconds (0-999999) Total: 64 bits = 8 bytes * SYYYYYYY.YYYYYYYY * .YYdddddh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff */ long intpart = buffer.getBeUlong40() - DATETIMEF_INT_OFS; // big-endian int frac = 0; switch (meta) { case 0: frac = 0; break; case 1: case 2: frac = buffer.getInt8() * 10000; break; case 3: case 4: frac = buffer.getBeInt16() * 100; break; case 5: case 6: frac = buffer.getBeInt24(); break; default: frac = 0; break; } String second = null; if (intpart == 0) { second = "0000-00-00 00:00:00"; } else { // 构造TimeStamp只处理到秒 long ymd = intpart >> 17; long ym = ymd >> 5; long hms = intpart % (1 << 17); // if (cal == null) cal = Calendar.getInstance(); // cal.clear(); // cal.set((int) (ym / 13), (int) (ym % 13) - 1, (int) (ymd // % (1 << 5)), (int) (hms >> 12), // (int) ((hms >> 6) % (1 << 6)), (int) (hms % (1 << 6))); // value = new Timestamp(cal.getTimeInMillis()); // second = String.format("%04d-%02d-%02d %02d:%02d:%02d", // (int) (ym / 13), // (int) (ym % 13), // (int) (ymd % (1 << 5)), // (int) (hms >> 12), // (int) ((hms >> 6) % (1 << 6)), // (int) (hms % (1 << 6))); StringBuilder builder = new StringBuilder(); builder.append(formatNumber((int) (ym / 13), 4)) .append('-') .append(formatNumber((int) (ym % 13), 2)) .append('-') .append(formatNumber((int) (ymd % (1 << 5)), 2)) .append(' ') .append(formatNumber((int) (hms >> 12), 2)) .append(':') .append(formatNumber((int) ((hms >> 6) % (1 << 6)), 2)) .append(':') .append(formatNumber((int) (hms % (1 << 6)), 2)); second = builder.toString(); } if (meta >= 1) { String microSecond = usecondsToStr(frac, meta); microSecond = microSecond.substring(0, meta); value = second + '.' + microSecond; } else { value = second; } javaType = Types.TIMESTAMP; length = 5 + (meta + 1) / 2; break; } case LogEvent.MYSQL_TYPE_TIME: { // MYSQL DataTypes: TIME // The range is '-838:59:59' to '838:59:59' // final int i32 = buffer.getUint24(); final int i32 = buffer.getInt24(); final int u32 = Math.abs(i32); if (i32 == 0) { value = "00:00:00"; } else { // if (cal == null) cal = Calendar.getInstance(); // cal.clear(); // cal.set(70, 0, 1, i32 / 10000, (i32 % 10000) / 100, i32 % // 100); // value = new Time(cal.getTimeInMillis()); // value = String.format("%s%02d:%02d:%02d", // (i32 >= 0) ? "" : "-", // u32 / 10000, // (u32 % 10000) / 100, // u32 % 100); StringBuilder builder = new StringBuilder(); if (i32 < 0) { builder.append('-'); } builder.append(formatNumber(u32 / 10000, 2)) .append(':') .append(formatNumber((u32 % 10000) / 100, 2)) .append(':') .append(formatNumber(u32 % 100, 2)); value = builder.toString(); } javaType = Types.TIME; length = 3; break; } case LogEvent.MYSQL_TYPE_TIME2: { /* * TIME low-level memory and disk representation routines * In-memory format: 1 bit sign (Used for sign, when on disk) 1 * bit unused (Reserved for wider hour range, e.g. for * intervals) 10 bit hour (0-836) 6 bit minute (0-59) 6 bit * second (0-59) 24 bits microseconds (0-999999) Total: 48 bits * = 6 bytes * Suhhhhhh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff */ long intpart = 0; int frac = 0; long ltime = 0; switch (meta) { case 0: intpart = buffer.getBeUint24() - TIMEF_INT_OFS; // big-endian ltime = intpart << 24; break; case 1: case 2: intpart = buffer.getBeUint24() - TIMEF_INT_OFS; frac = buffer.getUint8(); if (intpart < 0 && frac > 0) { /* * Negative values are stored with reverse * fractional part order, for binary sort * compatibility. Disk value intpart frac Time value * Memory value 800000.00 0 0 00:00:00.00 * 0000000000.000000 7FFFFF.FF -1 255 -00:00:00.01 * FFFFFFFFFF.FFD8F0 7FFFFF.9D -1 99 -00:00:00.99 * FFFFFFFFFF.F0E4D0 7FFFFF.00 -1 0 -00:00:01.00 * FFFFFFFFFF.000000 7FFFFE.FF -1 255 -00:00:01.01 * FFFFFFFFFE.FFD8F0 7FFFFE.F6 -2 246 -00:00:01.10 * FFFFFFFFFE.FE7960 Formula to convert fractional * part from disk format (now stored in "frac" * variable) to absolute value: "0x100 - frac". To * reconstruct in-memory value, we shift to the next * integer value and then substruct fractional part. */ intpart++; /* Shift to the next integer value */ frac -= 0x100; /* -(0x100 - frac) */ // fraclong = frac * 10000; } frac = frac * 10000; ltime = intpart << 24; break; case 3: case 4: intpart = buffer.getBeUint24() - TIMEF_INT_OFS; frac = buffer.getBeUint16(); if (intpart < 0 && frac > 0) { /* * Fix reverse fractional part order: * "0x10000 - frac". See comments for FSP=1 and * FSP=2 above. */ intpart++; /* Shift to the next integer value */ frac -= 0x10000; /* -(0x10000-frac) */ // fraclong = frac * 100; } frac = frac * 100; ltime = intpart << 24; break; case 5: case 6: intpart = buffer.getBeUlong48() - TIMEF_OFS; ltime = intpart; frac = (int) (intpart % (1L << 24)); break; default: intpart = buffer.getBeUint24() - TIMEF_INT_OFS; ltime = intpart << 24; break; } String second = null; if (intpart == 0) { second = "00:00:00"; } else { // 目前只记录秒，不处理us frac // if (cal == null) cal = Calendar.getInstance(); // cal.clear(); // cal.set(70, 0, 1, (int) ((intpart >> 12) % (1 << 10)), // (int) ((intpart >> 6) % (1 << 6)), // (int) (intpart % (1 << 6))); // value = new Time(cal.getTimeInMillis()); long ultime = Math.abs(ltime); intpart = ultime >> 24; // second = String.format("%s%02d:%02d:%02d", // ltime >= 0 ? "" : "-", // (int) ((intpart >> 12) % (1 << 10)), // (int) ((intpart >> 6) % (1 << 6)), // (int) (intpart % (1 << 6))); StringBuilder builder = new StringBuilder(); if (ltime < 0) { builder.append('-'); } builder.append(formatNumber((int) ((intpart >> 12) % (1 << 10)), 2)) .append(':') .append(formatNumber((int) ((intpart >> 6) % (1 << 6)), 2)) .append(':') .append(formatNumber((int) (intpart % (1 << 6)), 2)); second = builder.toString(); } if (meta >= 1) { String microSecond = usecondsToStr(Math.abs(frac), meta); microSecond = microSecond.substring(0, meta); value = second + '.' + microSecond; } else { value = second; } javaType = Types.TIME; length = 3 + (meta + 1) / 2; break; } case LogEvent.MYSQL_TYPE_NEWDATE: { /* * log_event.h : This enumeration value is only used internally * and cannot exist in a binlog. */ logger.warn("MYSQL_TYPE_NEWDATE : This enumeration value is " + "only used internally and cannot exist in a binlog!"); javaType = Types.DATE; value = null; /* unknown format */ length = 0; break; } case LogEvent.MYSQL_TYPE_DATE: { // MYSQL DataTypes: // range: 0000-00-00 ~ 9999-12-31 final int i32 = buffer.getUint24(); if (i32 == 0) { value = "0000-00-00"; } else { // if (cal == null) cal = Calendar.getInstance(); // cal.clear(); /* month is 0-based, 0 for january. */ // cal.set((i32 / (16 * 32)), (i32 / 32 % 16) - 1, (i32 % // 32)); // value = new java.sql.Date(cal.getTimeInMillis()); // value = String.format("%04d-%02d-%02d", i32 / (16 * 32), // i32 / 32 % 16, i32 % 32); StringBuilder builder = new StringBuilder(); builder.append(formatNumber(i32 / (16 * 32), 4)) .append('-') .append(formatNumber(i32 / 32 % 16, 2)) .append('-') .append(formatNumber(i32 % 32, 2)); value = builder.toString(); } javaType = Types.DATE; length = 3; break; } case LogEvent.MYSQL_TYPE_YEAR: { // MYSQL DataTypes: YEAR[(2|4)] // In four-digit format, values display as 1901 to 2155, and // 0000. // In two-digit format, values display as 70 to 69, representing // years from 1970 to 2069. final int i32 = buffer.getUint8(); // If connection property 'YearIsDateType' has // set, value is java.sql.Date. /* * if (cal == null) cal = Calendar.getInstance(); cal.clear(); * cal.set(Calendar.YEAR, i32 + 1900); value = new * java.sql.Date(cal.getTimeInMillis()); */ // The else, value is java.lang.Short. if (i32 == 0) { value = "0000"; } else { value = String.valueOf((short) (i32 + 1900)); } // It might seem more correct to create a java.sql.Types.DATE // value // for this date, but it is much simpler to pass the value as an // integer. The MySQL JDBC specification states that one can // pass a java int between 1901 and 2055. Creating a DATE value // causes truncation errors with certain SQL_MODES // (e.g."STRICT_TRANS_TABLES"). javaType = Types.VARCHAR; // Types.INTEGER; length = 1; break; } case LogEvent.MYSQL_TYPE_ENUM: { final int int32; /* * log_event.h : This enumeration value is only used internally * and cannot exist in a binlog. */ switch (len) { case 1: int32 = buffer.getUint8(); break; case 2: int32 = buffer.getUint16(); break; default: throw new IllegalArgumentException("!! Unknown ENUM packlen = " + len); } // logger.warn("MYSQL_TYPE_ENUM : This enumeration value is " // + "only used internally and cannot exist in a binlog!"); value = Integer.valueOf(int32); javaType = Types.INTEGER; length = len; break; } case LogEvent.MYSQL_TYPE_SET: { final int nbits = (meta & 0xFF) * 8; len = (nbits + 7) / 8; if (nbits > 1) { // byte[] bits = new byte[len]; // buffer.fillBytes(bits, 0, len); // 转化为unsign long switch (len) { case 1: value = buffer.getInt8(); break; case 2: value = buffer.getUint16(); break; case 3: value = buffer.getUint24(); break; case 4: value = buffer.getUint32(); break; case 5: value = buffer.getUlong40(); break; case 6: value = buffer.getUlong48(); break; case 7: value = buffer.getUlong56(); break; case 8: value = buffer.getUlong64(); break; default: throw new IllegalArgumentException("!! Unknown Set len = " + len); } } else { final int bit = buffer.getInt8(); // value = (bit != 0) ? Boolean.TRUE : Boolean.FALSE; value = bit; } javaType = Types.BIT; length = len; break; } case LogEvent.MYSQL_TYPE_TINY_BLOB: { /* * log_event.h : This enumeration value is only used internally * and cannot exist in a binlog. */ logger.warn("MYSQL_TYPE_TINY_BLOB : This enumeration value is " + "only used internally and cannot exist in a binlog!"); } case LogEvent.MYSQL_TYPE_MEDIUM_BLOB: { /* * log_event.h : This enumeration value is only used internally * and cannot exist in a binlog. */ logger.warn("MYSQL_TYPE_MEDIUM_BLOB : This enumeration value is " + "only used internally and cannot exist in a binlog!"); } case LogEvent.MYSQL_TYPE_LONG_BLOB: { /* * log_event.h : This enumeration value is only used internally * and cannot exist in a binlog. */ logger.warn("MYSQL_TYPE_LONG_BLOB : This enumeration value is " + "only used internally and cannot exist in a binlog!"); } case LogEvent.MYSQL_TYPE_BLOB: { /* * BLOB or TEXT datatype */ switch (meta) { case 1: { /* TINYBLOB/TINYTEXT */ final int len8 = buffer.getUint8(); byte[] binary = new byte[len8]; buffer.fillBytes(binary, 0, len8); value = binary; javaType = Types.VARBINARY; length = len8; break; } case 2: { /* BLOB/TEXT */ final int len16 = buffer.getUint16(); byte[] binary = new byte[len16]; buffer.fillBytes(binary, 0, len16); value = binary; javaType = Types.LONGVARBINARY; length = len16; break; } case 3: { /* MEDIUMBLOB/MEDIUMTEXT */ final int len24 = buffer.getUint24(); byte[] binary = new byte[len24]; buffer.fillBytes(binary, 0, len24); value = binary; javaType = Types.LONGVARBINARY; length = len24; break; } case 4: { /* LONGBLOB/LONGTEXT */ final int len32 = (int) buffer.getUint32(); byte[] binary = new byte[len32]; buffer.fillBytes(binary, 0, len32); value = binary; javaType = Types.LONGVARBINARY; length = len32; break; } default: throw new IllegalArgumentException("!! Unknown BLOB packlen = " + meta); } break; } case LogEvent.MYSQL_TYPE_VARCHAR: case LogEvent.MYSQL_TYPE_VAR_STRING: { /* * Except for the data length calculation, MYSQL_TYPE_VARCHAR, * MYSQL_TYPE_VAR_STRING and MYSQL_TYPE_STRING are handled the * same way. */ len = meta; if (len < 256) { len = buffer.getUint8(); } else { len = buffer.getUint16(); } if (isBinary) { // fixed issue #66 ,binary类型在binlog中为var_string /* fill binary */ byte[] binary = new byte[len]; buffer.fillBytes(binary, 0, len); javaType = Types.VARBINARY; value = binary; } else { value = buffer.getFullString(len, charsetName); javaType = Types.VARCHAR; } length = len; break; } case LogEvent.MYSQL_TYPE_STRING: { if (len < 256) { len = buffer.getUint8(); } else { len = buffer.getUint16(); } if (isBinary) { /* fill binary */ byte[] binary = new byte[len]; buffer.fillBytes(binary, 0, len); javaType = Types.BINARY; value = binary; } else { value = buffer.getFullString(len, charsetName); javaType = Types.CHAR; // Types.VARCHAR; } length = len; break; } case LogEvent.MYSQL_TYPE_JSON: { switch (meta) { case 1: { len = buffer.getUint8(); break; } case 2: { len = buffer.getUint16(); break; } case 3: { len = buffer.getUint24(); break; } case 4: { len = (int) buffer.getUint32(); break; } default: throw new IllegalArgumentException("!! Unknown JSON packlen = " + meta); } if (0 == len) { // fixed issue #1 by lava, json column of zero length has no // value, value parsing should be skipped value = ""; } else { int position = buffer.position(); Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1, charsetName); StringBuilder builder = new StringBuilder(); jsonValue.toJsonString(builder, charsetName); value = builder.toString(); buffer.position(position + len); } javaType = Types.VARCHAR; length = len; break; } case LogEvent.MYSQL_TYPE_GEOMETRY: { /* * MYSQL_TYPE_GEOMETRY: copy from BLOB or TEXT */ switch (meta) { case 1: len = buffer.getUint8(); break; case 2: len = buffer.getUint16(); break; case 3: len = buffer.getUint24(); break; case 4: len = (int) buffer.getUint32(); break; default: throw new IllegalArgumentException("!! Unknown MYSQL_TYPE_GEOMETRY packlen = " + meta); } /* fill binary */ byte[] binary = new byte[len]; buffer.fillBytes(binary, 0, len); /* Warning unsupport cloumn type */ logger.warn(String.format("!! Unsupport column type MYSQL_TYPE_GEOMETRY: meta=%d (%04X), len = %d", meta, meta, len)); javaType = Types.BINARY; value = binary; length = len; break; } default: logger.error(String.format("!! Don't know how to handle column type=%d meta=%d (%04X)", type, meta, meta)); javaType = Types.OTHER; value = null; length = 0; } return value; }
10. 处理完数据封装到rowdatbulider，然后再封装的到rowchanglebuider
11 封装为rowchage，再编码为entry（probuf）
同理，update和delete是一样的。只是tpye不一样。
5） XID_EVENT=16 把xid，和transactionend编码到enrty里
private Entry parseXidEvent(XidLogEvent event) { TransactionEnd transactionEnd = createTransactionEnd(event.getXid()); Header header = createHeader(binlogFileName, event.getHeader(), "", "", null); return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString()); }
event----》entry组装完毕。