mysqlbinlog日志上传集群HDFS

#!/bin/bash
# 
#
 
echo "执行节点"`hostname`
# 正 时
ip=$1
portc=4044
if [ $# -eq 4 ] ; then
  portc=$2
yearId=${3}
 dateId=${4}
 startTime=${4}" 00:00:00"
 endTime=${4}" 23:59:59"
else
   echo "输入参数有误： "
   exit -1
fi
 
sufix=`echo $ip | awk -F'.' '{print $NF}'`
username=lefin
platformname=tradeplatform
resultFile=result_resource_new_${sufix}
# 输入输出
inputPath=/data/lefin/${platformname}/input/${yearId}/${dateId}/
outputPath=/data/lefin/${platformname}/output/${yearId}/${dateId}/
# 数据库配置
source ./params.properties
hosts=$ip
port=$portc
echo  /usr/local/mariadb/bin/mysql -u${username} -p${password} -h${hosts} -P${port}
echo ############################begin###################################
for host in ${hosts} ; do
# 获取开始文件名
  binlogbashPath="le_finance-bin."
  startFile=`echo "show master status" | /usr/local/mariadb/bin/mysql -u${username} -p${password} -h${host} -P${port} |tail -1 | awk -F' ' '{print $1}'`
  lengthflist=`echo "show binary logs" | /usr/local/mariadb/bin/mysql -u${username} -p${password} -h${host} -P${port} |sed 1d | awk -F' ' '{print $1}'|wc -l`
if [ $lengthflist -gt 40 ] ; then
   validnumber=`echo $((${lengthflist}/2))`
else
  validnumber=${lengthflist}
fi
   
 # 获取抽取数据文件名
 flist=`echo "show binary logs" | /usr/local/mariadb/bin/mysql -u${username} -p${password} -h${host} -P${port} |sed 1d | awk -F' ' '{print $1}'| tail -n ${validnumber}`
 echo '###########开始拉取binlog日志 '`date '+%Y年%m月%d日 %H:%M:%S'`
 for blf in $flist ; do
    # 拉取binlog数据
    echo "/usr/local/mariadb/bin/mysqlbinlog -u${username} -p${password} -h${host} -P${port}  --read-from-remote-server --set-charset=utf8 --base64-output=DECODE-ROWS -v --start-datetime='${startTime}' --stop-datetime='${endTime}'  ${blf}  >> ./${resultFile}.sql"
    /usr/local/mariadb/bin/mysqlbinlog -u${username} -p${password} -h${host} -P${port}  --read-from-remote-server --set-charset=utf8 --base64-output=DECODE-ROWS -v --start-datetime="${startTime}" --stop-datetime="${endTime}" ${blf}  >> ./${resultFile}.sql
    if [[ $? -ne 0 ]]; then
            echo "获取binlog日志失败"
            exit
    else
            echo "获取binlog日志成功"
    fi
 done
done
echo '###########结束拉取binlog日志 '`date '+%Y年%m月%d日 %H:%M:%S'`
 
cc=`hadoop fs -ls /data/lefin/${platformname}/input/ | grep "/${yearId}"| wc -l`
if [ $cc -eq 0 ] ; then
   hadoop fs -mkdir -p /data/lefin/${platformname}/input/${yearId}/${dateId}
else
   cc=`hadoop fs -ls /data/lefin/${platformname}/input/${yearId}/ | grep "/${dateId}"| wc -l`
   if [ $cc -eq 0 ] ; then
     hadoop fs -mkdir /data/lefin/${platformname}/input/${yearId}/${dateId}
   fi
fi
 
echo '###########上传binlog日志至HDFS '`date '+%Y年%m月%d日 %H:%M:%S'`
# 上传binlog数据到hdfs
echo "hadoop fs -put -f ./${resultFile}.sql ${inputPath}"
hadoop fs -put -f ./${resultFile}.sql ${inputPath}
if [[ $? -ne 0 ]] ; then 
 echo "数据上传失败！"
 exit -1
fi    
echo '###########上传binlog日志至HDFS 结束'`date '+%Y年%m月%d日 %H:%M:%S'`