MySQL 向TiDB 实时迁移

可以借助如下工具将MySQL的数库备份恢复到TiDB，并借助sycner工具将增量的数据同步到TiDB。
1.支持分库分表的合并
2.支持全量+增量的数据同步方式.


缺点：
1.大数据量 超过100G的时候 loader导入的数据难以接受，速度慢
2.当有分库分表的时候，在源库做DDL操作的时候会导致syncer挂掉。

针对1 PingCAP开发tidb-lighning工具，针对问题2开发了TiDB-DM工具。
但是TiDB-DM工具配置相当复杂。

 

备份还原同步的工具：
MySQL的版本支持5.6和5.7，其他版本暂不支持。
--从MySQL备份：
1.备份指定的库
./mydumper -h 172.16.1.87 -u backup -p backup -P 3312 --regex '^(jp_bizuser\.|jp_admin\.|jp_partner\.)' -t 16 -F 64 -o /data/backup/
2.全库备份：
mydumper -h 172.16.1.87 -u backup -p backup -P 3312 -t 16 -F 64 -o /data/backup/
--还原：
#vim loader_trd.sh
#!/bin/bash
sed -i 's/CHARACTER SET utf8mb4//g' /data/backup/*-schema.sql      
sed -i 's/COLLATE utf8mb4_unicode_ci//g' /data/backup/*-schema.sql
/usr/local/tidb/bin/loader -h 10.10.58.42 -u root -P 4000 -s jp_admin -B AdminDB -d /data/backup -rm-checkpoint -t 64 -L error
/usr/local/tidb/bin/loader -h 10.10.58.42 -u root -P 4000 -s jp_bizuser -B BizUserDB -d /data/backup -rm-checkpoint -t 64 -L error
/usr/local/tidb/bin/loader -h 10.10.58.42 -u root -P 4000 -s jp_order_1 -B OrderDB -d /data/backup -rm-checkpoint -t 64 -L error
/usr/local/tidb/bin/loader -h 10.10.58.42 -u root -P 4000 -s jp_order_2 -B OrderDB -d /data/backup -rm-checkpoint -t 64 -L error
/usr/local/tidb/bin/loader -h 10.10.58.42 -u root -P 4000 -s jp_order_3 -B OrderDB -d /data/backup -rm-checkpoint -t 64 -L error

--同步：
查看metadata 文件：
# cat /root/backup/metadata 
Started dump at: 2019-03-14 14:11:29
SHOW MASTER STATUS:
        Log: trading_bin.000009
        Pos: 94637832
        GTID:25cd6e33-21fc-11e9-8f4b-005056bec689:1-1426,
c62e3ab0-0f96-11e8-818a-005056beb362:1-82382

SHOW SLAVE STATUS:
        Host: 172.16.1.86
        Log: trading_bin.000315
        Pos: 95455532
        GTID:25cd6e33-21fc-11e9-8f4b-005056bec689:1-1426,
c62e3ab0-0f96-11e8-818a-005056beb362:1-82382

Finished dump at: 2019-03-14 14:16:38

选择从slave库继续同步。

# cat syncer_trd.meta 
binlog-name = "trading_bin.000315"
binlog-pos = 95455532
binlog-gtid = "25cd6e33-21fc-11e9-8f4b-005056bec689:1-1426,
c62e3ab0-0f96-11e8-818a-005056beb362:1-82382"

#cat syncer_trad.toml

meta = "/data/tidb/tools/bin/syncer_trd.meta"
worker-count = 32
batch = 10
status-addr = "172.16.4.172:10086"
log-file = "/data/tidb/tools/bin/log/syncer_trad.log"


replicate-ignore-db = ["~^wx.*","maxwell","wuhan","bi_trading","rt_bigscreen","tempdb","~^jp_b.*","jp_else","~^jp_pa.*","~^jp_prod.*","jp_rule","~^jp_s.*"]

[[replicate-do-table]]
db-name = "jp_order_1"
tbl-name = "orders"

[[replicate-do-table]]
db-name = "jp_order_2"
tbl-name = "orders"

[[replicate-do-table]]
db-name = "jp_order_3"
tbl-name = "orders"

[[replicate-do-table]]
db-name = "jp_order_1"
tbl-name = "orderitem"

[[replicate-do-table]]
db-name = "jp_order_2"
tbl-name = "orderitem"

[[replicate-do-table]]
db-name = "jp_order_3"
tbl-name = "orderitem"

[[replicate-do-table]]
db-name = "jp_promotion"
tbl-name = "compositeproduct"

[[replicate-do-table]]
db-name = "jp_promotion"
tbl-name = "discountpromotion"

[[replicate-do-table]]
db-name = "jp_promotion"
tbl-name = "gatherorder"

[[replicate-do-table]]
db-name = "jp_promotion"
tbl-name = "grouppurchase"

[[replicate-do-table]]
db-name = "jp_promotion"
tbl-name = "timelimitpromotion"

[[replicate-do-table]]
db-name = "jp_admin"
tbl-name = "city"

[from]
host = "172.16.4.170"
user = "repl"
password = "xyz"
port = 3312

[to]
host = "172.16.4.172"
user = "root"
password = "xyz"
port = 4000


--启动配置文件：
# cat start_syncer_trd.sh 
#!/bin/bash
/data/tidb/tools/bin/syncer -config /data/tidb/tools/bin/syncer_trd.toml &

# cat stop_syncer_trd.sh 
#!/bin/bash
kill `ps -ef|grep syncer |grep 'syncer_trd.toml' | grep -v grep | awk '{print $2}'`
echo "syncer is stoped!"