shell mysql数据实时同步脚本

1. 背景 

 公司大佬需要报表实时更新，从业务报表同步到数据仓库，可根据需要设置同步时间几个crontab 或者while true,用shell脚本获取增量数据，实时更新，业务系统存在物理删除的场景，这就需要去监控业务库删除操作，自己写了粗糙的脚本。本来是可以用otter 工具同步实现，但是好资源，我们也就只有几张表而已。

2. 前提

 数据库配置文件：

[hxw@test1 Shell]$ cat  dbConfig.sh
ADMIN_DB="mysql -s -h127.0.0.1 -P3306 -uXXXXXXX -pXXXXXXXX GKIY"

mysql配置表：

3.代码

---------------------------数据库代码----------------------
CREATE TABLE `t_sys_real_sync_config` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `source_db` varchar(50) DEFAULT NULL,
  `source_table` varchar(250) DEFAULT NULL,
  `source_delete_cnt` varchar(50) DEFAULT NULL,
  `target_db` varchar(50) DEFAULT NULL,
  `target_table` varchar(250) DEFAULT NULL,
  `source_sql` varchar(5000) DEFAULT NULL,
  `target_sql` varchar(5000) DEFAULT NULL,
  `s_id` varchar(50) DEFAULT '' COMMENT '源表主键id',
  `t_id` varchar(50) DEFAULT '' COMMENT '仓库表主键',
  `status` tinyint(2) DEFAULT '0',
  `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `updatetime` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8;

-------------------------------shell------------------------------
#!/bin/bash
#date :2018-03-02
#create by :houxiaowei
#comment : used  to sysnc the real data  from source db  to the target db ,just for phyica detele SQL  from the source DB 

#import the DB configurtion
. /data/stat/lib/bondlib.sh

script_path=`dirname $0`
script_name=`basename $0`

tempFile=${script_path}/${script_name/.sh/_temp}
s_priId_temp=${script_path}/s_priId_temp.temp
t_priId_temp=${script_path}/t_priId_temp.temp
logFile_temp=${script_path}/error_temp.log
logFile=${script_path}/${script_name/.sh/.log}

#the global vairables of the mysql Handler_delete ,it is The number of times that rows have been deleted from tables. 
#the global vairables of the mysql com_delete ,The Com_delete statement counter variables indicate the number of times each delete statement has been executed.
# the Handler_delete increase just when the real row is delete . but com_delete is delete when the delete sql is executed,do not care the affacted the rows.

#the delete count  in the target DB:

${REPORT_DB}  -s -e "select id, source_db,ifnull(source_delete_cnt,0) as  source_delete_cnt ,source_table,target_db,target_table,source_sql,target_sql,s_id,t_id from SYS.t_sys_real_sync_config  where status=1" 2>${logFile} |sed 's/\t/|/g' 1>${tempFile}


################-----------------------------------------log part---------------------------------------------------------
#usage:  log_write "$FUNCNAME" "$LINENO"

log_write()
{
#判断删除成功失败
file_cnt=$(cat ${logFile_temp} |grep -v Warning |wc -l)


if [ $file_cnt -gt 0 ];then
   
   dateT=`date "+%Y-%m-%d %T"`
   
   echo "$dateT [LINE]:$2 Function:$1 executed faild!!" >${logFile}
   exit 0;
fi
 
}

###############----------------------------------------- end  -----------------------------------------------------------


#uasge : do_check $source_db $target_db $source_sql $target_sql
do_check()
{


#取变量的地址,eval 解决动态变量问题
 eval local sourcedb="$"$1
 eval local targetdb="$"$2
 local sourcesql=$3
 local targetsql=$4


V_Source_Del_cnt=$(${sourcedb} -N -e "show global status like 'Handler_delete' " 2>${logFile_temp}|cut  -f2)


log_write "$FUNCNAME" "$LINENO"


#直接重定向解决Mysql因为安全考虑输出的Warning。
v_Source_Total_cnt= $sourcedb -s -e "$sourcesql" 2>${logFile_temp};


log_write "$FUNCNAME" "$LINENO"


V_Target_Total_cnt= ${targetdb} -s -e "$targetsql" 2>${logFile_temp};


log_write "$FUNCNAME" "$LINENO"




#如果全局变量相等则不执行更新，否则比较表记录数据
if [ ${V_Source_Del_cnt} -eq ${source_delete_cnt} ];then
        
        return 0;


elif [ ${V_Source_Del_cnt} -gt ${source_delete_cnt} ];then


         #源表记录数跟仓库记录数不一致，执行删除，更新数据库
        if [ $v_Source_Total_cnt != $V_Target_Total_cnt ];then     
        
          do_delete ${target_db} ${target_table} ${t_id}


          do_feedback "${V_Source_Del_cnt}" "${source_db}"


        else


          return 0


        fi
else
        do_feedback "${V_Source_Del_cnt}" "${source_db}"


fi


}


#执行delete操作
#参数 do_delete [targetdb] [targettable] [t_id]
do_delete()
{
 eval local targetdb="$"$1
 eval local targetable=$2
 eval local t_id=$3


do_findout_diff ${source_db} ${source_table} ${s_id} ${target_db} ${target_table} ${t_id}


$targetdb  -e "delete from ${targetable} where ${t_id} in (${IDS})" 2>${logFile_temp}


log_write "$FUNCNAME" "$LINENO" 


do_feedback


}


##执行回写功能
do_feedback()
{

 ${REPORT_DB}  -s -e "update SYS.t_sys_real_sync_config set source_delete_cnt=${V_Source_Del_cnt} where id=${c_id}" >${logFile_temp}
 
 log_write "$FUNCNAME" "$LINENO"


}

#找出不同主键的ID，参数源库表，主键id，连接方式，返回要删除主键列表用逗号分隔。
#usage: do_findout_diff [sourcedb] [sourcetable] [s_id] [targetdb] [targettable] [t_id]
do_findout_diff()
{
  ##接受参数,local 是变量只能在函数内部使用
 eval local sourcedb="$"$1
 eval local sourcetable=$2
 eval local s_id=$3
 eval local targetdb="$"$4
 eval local targettable=$5
 eval local t_id=$6


$sourcedb  -e "select ${s_id} from ${sourcetable}" 2>${logFile_temp} 1>${s_priId_temp}


log_write "$FUNCNAME" "$LINENO"


${targetdb}  -e "select ${t_id} from ${targettable}" 2>${logFile_temp} 1>${t_priId_temp}


log_write "$FUNCNAME" "$LINENO"

#id 结果集，函数定义为全局变量

#join 有坑，查出来有问题
#IDS=$(join  -v 2 -o 2.1 ${s_priId_temp}  ${t_priId_temp}  --nocheck-order|xargs echo |sed 's/ /,/g')


#awk执行
IDS=$( awk 'NR==FNR{a[$1]=$0;next}{if(!($1 in a)) print $1}' ${s_priId_temp} ${t_priId_temp} |xargs echo |sed 's/ /,/g')


if [ "$IDS" = "" ];then
  exit 0
fi

}

while read line
do 
        num=1
        for name in c_id source_db source_delete_cnt source_table target_db target_table source_sql target_sql s_id t_id
          do
                #echo -n "$name="
                #echo -n "$(echo $line|cut -d "|"  -f${num})"
                #动态给变量赋值
                eval "$name='`echo -n $(echo $line|cut -d "|"  -f${num} )`'"
                #echo ${source_db}  "${target_db}" "${source_sql}" "${target_sql}"
                #开始检测业务库和数据仓库数据同步
                #do_check "${source_db}" "${target_db}" "${source_sql}" "${target_sql}"


                num=`expr $num + 1`
          done
                do_check "${source_db}" "${target_db}" "${source_sql}" "${target_sql}"


done < ${tempFile}