生产系统数据实时同步到rds数据仓库的一个样例

1、环境说明
生产系统数据库是阿里云mysql型rds数据库，目标数据仓库也是阿里云mysql型rds数据库。
由于数据高效抽取的需要，生产系统表字段timeline定义为记录新增或更新的时间戳，并且此字段需要添加索引。

2、存储数据抽取时间的表

CREATE TABLE `real_time_maintenance` (
  `id` int(11) DEFAULT NULL,
  `last_run_time` timestamp NULL DEFAULT NULL,
  `update_time` timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

3、源、目标及中间临时表结构

<strong>CREATE TABLE `real_99_dingdan_all` (
  `dingd_id` bigint(20) unsigned NOT NULL,
  `status` tinyint(3) unsigned NOT NULL DEFAULT '0',
  `type` tinyint(3) unsigned NOT NULL DEFAULT '0',
  `paytype` tinyint(3) unsigned DEFAULT '0',
  `paystatus` tinyint(3) unsigned DEFAULT '0',
  `pay_trade_no` char(180) DEFAULT NULL,
  `source` tinyint(3) unsigned DEFAULT '0',
  `consumption_type` tinyint(3) NOT NULL DEFAULT '0' COMMENT '消费方式',
  `suspicious` tinyint(3) NOT NULL DEFAULT '0',
  `sid` int(10) unsigned NOT NULL DEFAULT '0',
  `site_id` int(10) unsigned NOT NULL,
  `dorm_id` int(10) DEFAULT NULL,
  `dormentry_id` int(10) DEFAULT NULL,
  `shop_id` int(11) unsigned NOT NULL DEFAULT '0' COMMENT '店铺id',
  `uid` bigint(20) unsigned NOT NULL,
  `service_eva` tinyint(3) DEFAULT NULL,
  `delivery_eva` tinyint(3) DEFAULT NULL,
  `food_eva` tinyint(3) DEFAULT NULL,
  `food_num` smallint(5) unsigned DEFAULT '0',
  `food_amount` decimal(10,2) unsigned NOT NULL DEFAULT '0.00',
  `ship_fee` decimal(10,1) unsigned DEFAULT '0.0',
  `coupon_discount` decimal(10,2) unsigned DEFAULT '0.00',
  `promotion_discount` decimal(10,2) unsigned DEFAULT '0.00',
  `discount` decimal(10,1) unsigned DEFAULT '0.0',
  `dingd_amount` decimal(10,2) unsigned DEFAULT '0.00',
  `delivery_id` int(10) unsigned DEFAULT NULL,
  `add_time` timestamp NULL,
  `confirm_time` timestamp NULL DEFAULT '0000-00-00 00:00:00',
  `send_time` timestamp NULL DEFAULT '0000-00-00 00:00:00',
  `expect_date` int(10) unsigned DEFAULT NULL,
  `delivery_type` tinyint(3) DEFAULT '0',
  `expect_time` timestamp NULL DEFAULT '0000-00-00 00:00:00',
  `expect_timeslot` varchar(48) DEFAULT '',
  `dingd_mark` int(10) DEFAULT NULL,
  `uname` varchar(120) DEFAULT '',
  `portrait` varchar(480) DEFAULT '',
  `phone` varchar(60) DEFAULT '',
  `phone_addr` varchar(60) DEFAULT NULL,
  `buy_times` int(10) DEFAULT NULL,
  `address1` varchar(120) DEFAULT '',
  `address2` varchar(300) DEFAULT '',
  `dormitory` varchar(180) DEFAULT NULL,
  `time_deliver` tinyint(3) unsigned DEFAULT NULL,
  `credit` smallint(5) unsigned DEFAULT '0',
  `ip` varchar(90) DEFAULT '',
  `coupon_code` varchar(90) DEFAULT '',
  `feature` varchar(180) DEFAULT NULL,
  `remark` varchar(450) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `evaluation` varchar(900) DEFAULT '',
  `expect_start_time` timestamp NULL DEFAULT '0000-00-00 00:00:00',
  `expect_end_time` timestamp NULL DEFAULT '0000-00-00 00:00:00',
  `timeline` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  `rn` tinyint(1) DEFAULT NULL,
  KEY `idx_99_dingdan_dingd_id` (`dingd_id`) USING BTREE,
  KEY `idx_99_dingdan_2` (`status`,`uid`,`add_time`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;</strong>

注：表99_dingdan、real_99_dingdan_all、real_99_dingdan_3min表结构完全相同，只不过定义用来存储不同的数据内容。

4、实时数据同步的shell脚本
[hs@master rds_real]$ pwd
/home/hs/opt/Log_Data/rds_real

[hs@master rds_real]$ cat src2rds_99_dingdan.sh

#!/bin/bash
export last_time=`/usr/local/bin/mysql -hdwipaddress.rds.aliyuncs.com -udatauser -piloveyou -N -e "use tmp;select last_run_time from real_time_maintenance where id=1;"`
/usr/local/bin/mysql -hdwipaddress.rds.aliyuncs.com -udatauser -piloveyou -N -e "use tmp;update real_time_maintenance set last_run_time=now() where id=1;"
export this_time=`/usr/local/bin/mysql -hdwipaddress.rds.aliyuncs.com -udatauser -piloveyou -N -e "use tmp;select last_run_time from real_time_maintenance where id=1;"`

#99_dingdan data proc ...
/usr/local/bin/mysql -hsrcipaddress.rds.aliyuncs.com -udbreader -piloveyou -N -e"select CONCAT(ifnull(\`dingd_id\`,''),'|',ifnull(\`status\`,''),'|',ifnull(\`type\`,''),'|',ifnull(\`paytype\`,''),'|',ifnull(\`paystatus\`,''),'|',ifnull(\`pay_trade_no\`,''),'|',ifnull(\`source\`,''),'|',ifnull(\`consumption_type\`,''),'|',ifnull(\`suspicious\`,''),'|',ifnull(\`sid\`,''),'|',ifnull(\`site_id\`,''),'|',ifnull(\`dorm_id\`,''),'|',ifnull(\`dormentry_id\`,''),'|',ifnull(\`shop_id\`,''),'|',ifnull(\`uid\`,''),'|',ifnull(\`service_eva\`,''),'|',ifnull(\`delivery_eva\`,''),'|',ifnull(\`food_eva\`,''),'|',ifnull(\`food_num\`,''),'|',ifnull(\`food_amount\`,''),'|',ifnull(\`ship_fee\`,''),'|',ifnull(\`coupon_discount\`,''),'|',ifnull(\`promotion_discount\`,''),'|',ifnull(\`discount\`,''),'|',ifnull(\`dingd_amount\`,''),'|',ifnull(\`delivery_id\`,''),'|',ifnull(from_unixtime(\`add_time\`,'%Y-%m-%d %H:%i:%s'),''),'|',ifnull(from_unixtime(\`confirm_time\`,'%Y-%m-%d %H:%i:%s'),''),'|',ifnull(from_unixtime(\`send_time\`,'%Y-%m-%d %H:%i:%s'),''),'|',ifnull(\`expect_date\`,''),'|',ifnull(\`delivery_type\`,''),'|',ifnull(from_unixtime(\`expect_time\`,'%Y-%m-%d %H:%i:%s'),''),'|',ifnull(replace(replace(replace(\`expect_timeslot\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(\`dingd_mark\`,''),'|',ifnull(replace(replace(replace(\`uname\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`portrait\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`phone\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`phone_addr\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(\`buy_times\`,''),'|',ifnull(replace(replace(replace(\`address1\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`address2\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`dormitory\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(\`time_deliver\`,''),'|',ifnull(\`credit\`,''),'|',ifnull(replace(replace(replace(\`ip\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`coupon_code\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`feature\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`remark\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(replace(replace(replace(\`evaluation\`,char(13),''),char(10),''),'|',''),''),'|',ifnull(from_unixtime(\`expect_start_time\`,'%Y-%m-%d %H:%i:%s'),''),'|',ifnull(from_unixtime(\`expect_end_time\`,'%Y-%m-%d %H:%i:%s'),''),'|',ifnull(\`timeline\`,'')) from db59store.99_dingdan where timeline between str_to_date('$last_time','%Y-%m-%d %H:%i:%s') and str_to_date('$this_time','%Y-%m-%d %H:%i:%s');" >/home/hs/opt/dw-etl/data/db59store.99_dingdan_3min.dat
/usr/local/bin/mysql -hdwipaddress.rds.aliyuncs.com -udatauser -piloveyou -e "use tmp;drop table if exists real_99_dingdan_3min;create table real_99_dingdan_3min as select * from tmp.real_99_dingdan_all where 1=2;load data local infile '/home/hs/opt/dw-etl/data/db59store.99_dingdan_3min.dat' into table real_99_dingdan_3min fields terminated by '|' enclosed by '' lines terminated by '\n' ignore 0 lines;alter table tmp.real_99_dingdan_3min add index idx_99_dingdan_dingd_id (dingd_id) using btree;delete ca.* from tmp.real_99_dingdan_all ca left join tmp.real_99_dingdan_3min i on ca.dingd_id = i.dingd_id where i.dingd_id is not null;insert into tmp.real_99_dingdan_all select * from tmp.real_99_dingdan_3min;"

5、定时调度crontab内容
[hs@master rds_real]$ crontab -l
0 * * * * sh /home/hs/opt/Log_Data/rds_real/src2rds_99_dingdan.sh

注：此处定义为每小时进行一次数据的同步处理；实际上该方案支持更短时间间隔的数据同步处理，前提是脚本在相应的间隔时间内能够跑完。