2019独角兽企业重金招聘Python工程师标准>>> 
#!/bin/bash
#tanj
#2017-09-11
DATE=`date +%Y-%m-%d`
TIME=`date +%H:%M:%S`
echo "$DATE $TIME-----------------start-----------------"
if [ ! -n "$1" ] ;then
echo "#you have not input a mysqldb!"
exit 0;
else
echo "#the mysqldb you input is $1";
param1=$1;
fi
if [ ! -n "$2" ] ;then
echo "#you have not input a pgid!"
exit 0;
else
echo "#the pgid you input is $2";
pgid=$2;
fi
#if flag is 0-- work is successful-- so, insert result
#if flag is not 0-- work is failed-- so, not insert result
#flag=1 --createTable is failed
#flag=2 --import is failed
#flag=3 --appendData is failed
#flag=4 --insertNum is failed
#flag=5 --connect is failed
#flag=6 --tuomin is failed
#flag=7 --tuominsql is failed
flag=10
function updateFlag() {
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set end_time='$DATE $TIME',result='$result',resultcode='$flag',status='end' where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;"
}
function createTable() {
echo "-----------------create table-------------------"
CREATEHOST=$HOST
CREATEPORT=$PORT
CREATEUSER=$USER
CREATEPASSWD=$PASSWD
CREATEMYSQLDB=$MYSQL_DB
CREATEHIVEDB=$HIVE_DB
CREATELOCATION=$HDFS_DIR
CREATEMYSQLTABLE=$HIVE_TABLE
CREATEHIVETABLE=$HIVE_TABLE
echo "#mysql -h $CREATEHOST -u $CREATEUSER -p$CREATEPASSWD -N -e \" use $CREATEMYSQLDB ; describe $CREATEMYSQLTABLE ;\" |awk '{print \$1,\$2}' ";
Table_Fields=`mysql -h ${CREATEHOST} -u ${CREATEUSER} -p${CREATEPASSWD} -N -e " use $CREATEMYSQLDB ; describe ${CREATEMYSQLTABLE} ;" |awk '{print $1,$2}' `
echo "${Table_Fields}";
Fields_Split=(${Table_Fields//,,,/ })
Fields_Len=${#Fields_Split[@]}
echo "$Fields_Len"
create_Table_Str="CREATE TABLE IF NOT EXISTS ${CREATEHIVEDB}.${CREATEHIVETABLE} ("
for (( createI=0; createI < $Fields_Len-1; createI=createI+2 ))
do
field=${Fields_Split[$createI]}
fieldType="string"
field_Type=${Fields_Split[$createI+1]}
echo $field_Type |grep -q "int"
if [ $? -eq 0 ]
then
fieldType="int"
fi
echo $field_Type | grep -q "decimal"
if [ $? -eq 0 ]
then
fieldType=$field_Type
fi
echo $field_Type | grep -q "double"
if [ $? -eq 0 ]
then
fieldType="decimal(18,4)"
fi
echo $field_Type | grep -q "double("
if [ $? -eq 0 ]
then
fieldType=${field_Type//double/decimal}
fi
echo $field_Type | grep -q "double(" |grep -q ",0)"
if [ $? -eq 0 ]
then
fieldType=${fieldType//,0/,4}
fi
echo $field_Type | grep -q "bit"
if [ $? -eq 0 ]
then
fieldType="boolean"
fi
create_Table_Str=${create_Table_Str}'`'$field'` '$fieldType','
done
create_Table_Str=${create_Table_Str}" ROW format delimited fields terminated BY '\001' stored AS textfile LOCATION '${CREATELOCATION}/${CREATEHIVETABLE}';"
echo ${create_Table_Str//, ROW format delimited fields/)ROW format delimited fields}
impala-shell --quiet -B -q "${create_Table_Str//, ROW format delimited fields/)ROW format delimited fields}"
if [ "$?" == "0" ]; then
flag=0;
TIME=`date +%H:%M:%S`
echo " $DATE $TIME $HIVE_TABLE createTable is successful"
else
flag=1;
TIME=`date +%H:%M:%S`
result="$DATE $TIME $HIVE_TABLE createTable is failed"
echo $result
updateFlag
fi
impala-shell --quiet -B -q " invalidate metadata ${CREATEHIVEDB}.${CREATEHIVETABLE} ";
}
function insertNum() {
echo "-----------------insert result-------------------"
TIME=`date +%H:%M:%S`
YESTERDAY=`date -d "1 day ago" +%Y-%m-%d`
echo "#impala-shell -q \" REFRESH $HIVE_DB.$HIVE_TABLE ; \";";
impala-shell -q " REFRESH $HIVE_DB.$HIVE_TABLE ; ";
echo "#impala-shell -q ' select count(1) from $HIVE_DB.$HIVE_TABLE ' | cat|grep -v count |awk '{print $2}'";
hive_num=`impala-shell -q " select count(1) from $HIVE_DB.$HIVE_TABLE " | cat|grep -v count |awk '{print $2}' |tail -n -2|head -n 1`
if [ $hive_num -ge 0 ]; then
echo $hive_num;
else
hive_num=-1
fi
TIME=`date +%H:%M:%S`
echo "#impala-shell -q \"insert into data_market.last_process_time(hivedb,mysqldb,tablename,status,synctime,hive_count_num,datatime) values('$HIVE_DB','$MYSQL_DB','$HIVE_TABLE','$flag','$DATE$TIME',$hive_num,'$YESTERDAY');\";";
impala-shell -q " insert into data_market.last_process_time(hivedb,mysqldb,tablename,status,synctime,hive_count_num,datatime) values('$HIVE_DB','$MYSQL_DB','$HIVE_TABLE','$flag','$DATE $TIME',$hive_num,'$YESTERDAY') ; ";
if [ "$?" == "0" ]; then
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set hive_count_num=$hive_num where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;"
TIME=`date +%H:%M:%S`
echo " $DATE $TIME MYSQL_DB=$MYSQL_DB MYSQL_TABLE=$MYSQL_TABLE insert into data_market.last_process_time is successful";
else
flag=4;
TIME=`date +%H:%M:%S`
result="$DATE $TIME MYSQL_DB=$MYSQL_DB MYSQL_TABLE=$MYSQL_TABLE insert into data_market.last_process_time is failed"
echo $result
updateFlag
fi
}
function appendData() {
echo "-----------------append table-------------------"
appendpgid=${pgid}1
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e \"use $db_conf;select mysqldb,check_column,last_value,host,port,user,password,sqoop_temp_dir,hdfs_dir,num_mappers from $table_conf where is_enabled=0 and tablename='$MYSQL_TABLE' and mysqldb='$MYSQL_DB' and pgid=${appendpgid} limit 1;\" |cat |grep -v mysqldb"
appendinfo=`mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e "use $db_conf;select mysqldb,check_column,last_value,host,port,user,password,sqoop_temp_dir,hdfs_dir,num_mappers from $table_conf where is_enabled=0 and tablename='$MYSQL_TABLE' and mysqldb='$MYSQL_DB' and pgid=${appendpgid} limit 1 ;" |cat |grep -v mysqldb`;
if [ "$?" == "0" ]; then
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e \"use $db_conf;UPDATE $table_conf SET result='',status='running',updatetime='$DATE',start_time='$DATE $TIME' WHERE mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=${appendpgid};\""
mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e "use $db_conf;UPDATE $table_conf SET result='',status='running',updatetime='$DATE',start_time='$DATE $TIME' WHERE mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=${appendpgid};"
appendinfosplit=(${appendinfo//,,/ })
appendlen=${#appendinfosplit[@]}
APPENDHOST=${appendinfosplit[3]}
APPENDPORT=${appendinfosplit[4]}
APPENDUSER=${appendinfosplit[5]}
APPENDPASSWD=${appendinfosplit[6]}
APPENDCOLUMN=${appendinfosplit[1]}
APPENDMYSQL_DB=${appendinfosplit[0]}
APPENDLASTVALUE=${appendinfosplit[2]}
APPENDTEMPDIR=${appendinfosplit[7]}
APPENDHDFS_DIR=${appendinfosplit[8]}
APPENDNUMMAPPERS=${appendinfosplit[9]}
echo $APPENDHOST;
echo $APPENDPORT;
echo $APPENDUSER;
echo $APPENDPASSWD;
echo $APPENDCOLUMN;
echo $APPENDMYSQL_DB;
echo $APPENDLASTVALUE;
echo $APPENDTEMPDIR;
echo $APPENDHDFS_DIR;
echo $APPENDNUMMAPPERS;
APPENDTEMPDIR=$APPENDTEMPDIR/$HIVE_TABLE
#APPENDLASTVALUE=`impala-shell --quiet -B -q "SELECT max(systime) FROM $HIVE_DB.$HIVE_TABLE LIMIT 1"`
echo "mysql -h ${APPENDHOST} -u$APPENDUSER -p$APPENDPASSWD -e \"use $APPENDMYSQL_DB;select * from $HIVE_TABLE limit 1\"";
mysql -h ${APPENDHOST} -u$APPENDUSER -p$APPENDPASSWD -e "use $APPENDMYSQL_DB;select * from $HIVE_TABLE limit 1"
if [ "$?" == "0" ]; then
echo "#sqoop import --connect jdbc:mysql://${APPENDHOST}:$APPENDPORT/$APPENDMYSQL_DB --username $APPENDUSER --password '$APPENDPASSWD' \
--table $HIVE_TABLE --incremental append --check-column $APPENDCOLUMN --outdir /root/tanj/sh/java \
--hive-import --hive-database $HIVE_DB --hive-table $HIVE_TABLE \
--target-dir $APPENDTEMPDIR --delete-target-dir --fields-terminated-by '\001' \
--hive-drop-import-delims --null-string '\\\\N' --null-non-string '\0' -m $APPENDNUMMAPPERS --last-value '${APPENDLASTVALUE};';";
sqoop import --connect jdbc:mysql://${APPENDHOST}:$APPENDPORT/$APPENDMYSQL_DB --username $APPENDUSER --password ''${APPENDPASSWD}'' \
--table $HIVE_TABLE --incremental append --check-column $APPENDCOLUMN --outdir /root/tanj/sh/java \
--hive-import --hive-database $HIVE_DB --hive-table $HIVE_TABLE \
--target-dir $APPENDTEMPDIR --delete-target-dir --fields-terminated-by "\001" \
--hive-drop-import-delims --null-string '\\N' --null-non-string "\0" -m $APPENDNUMMAPPERS --last-value "${APPENDLASTVALUE};";
if [ "$?" == "0" -a "$flag" == "0" ]; then
flag=0;
TIME=`date +%H:%M:%S`
echo " $DATE $TIME $HIVE_TABLE append is successful"
else
flag=3;
TIME=`date +%H:%M:%S`
result="$DATE $TIME $HIVE_TABLE append is failed"
echo $result
updateFlag
fi
impala-shell -q " REFRESH $HIVE_DB.$HIVE_TABLE ";
fi
impala-shell --quiet -B -q " invalidate metadata $HIVE_DB.$HIVE_TABLE ";
TIME=`date +%H:%M:%S`
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -e \"use $db_conf; update $table_conf set updatetime='$DATE $TIME',end_time='$DATE $TIME',status='end',run_time=TIMEDIFF($TIME,$startTime) where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=${appendpgid};\";";
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set updatetime='$DATE $TIME',end_time='$DATE $TIME',status='end',run_time=TIMEDIFF($TIME,$startTime) where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=${appendpgid};"
else
echo " $DATE $TIME $HIVE_TABLE not exists append "
fi
}
function runSqoop() {
#-------------------- create datebases -------------------------
echo "impala-shell -q \" CREATE DATABASE IF NOT EXISTS ${HIVE_DB} \";";
impala-shell -q " CREATE DATABASE IF NOT EXISTS ${HIVE_DB} ";
echo "-----------------create table-------------------"
impala-shell -q " drop table $HIVE_DB.$HIVE_TABLE "
createTable;
impala-shell --quiet -B -q " REFRESH $HIVE_DB.$HIVE_TABLE "
echo "#mysql -h $HOST -u$USER -p$PASSWD -e\" use $MYSQL_DB ; select * from $MYSQL_TABLE limit 1 \";";
mysql -h $HOST -u$USER -p$PASSWD -e" use $MYSQL_DB ; select * from $MYSQL_TABLE limit 1 "
if [ "$?" == "0" -a "$flag" == "0" ]; then
echo "-----------------import table-------------------";
#echo $TEMPDIR | grep -q "$sqoop_temp_dir";
# if [ $? -eq 0 ]
# then
# echo "hdfs dfs -rm -R $TEMPDIR";
# hdfs dfs -rm -R $TEMPDIR
# fi
echo "#sqoop import --connect jdbc:mysql://$HOST:$PORT/$MYSQL_DB --username $USER --password '$PASSWD' --table $MYSQL_TABLE \
--hive-import --hive-overwrite --hive-database $HIVE_DB --fields-terminated-by '\001' --target-dir $TEMPDIR --delete-target-dir --outdir /root/tanj/sh/java \
--hive-drop-import-delims --null-string '\\\\N' --null-non-string '\0' -m $NUMMAPPERS";
PASSWD1="'$PASSWD'"
sqoop import --connect jdbc:mysql://$HOST:$PORT/$MYSQL_DB --username $USER --password ''${PASSWD}'' --table $MYSQL_TABLE \
--hive-import --hive-overwrite --hive-database $HIVE_DB --fields-terminated-by '\001' --target-dir $TEMPDIR --delete-target-dir --outdir /root/tanj/sh/java \
--hive-drop-import-delims --null-string '\\N' --null-non-string '\0' -m $NUMMAPPERS
if [ "$?" == "0" ]; then
TIME=`date +%H:%M:%S`
echo " $DATE $TIME sqoop import $MYSQL_DB.$MYSQL_TABLE is successful";
else
flag=2;
TIME=`date +%H:%M:%S`
result="$DATE $TIME sqoop import $MYSQL_DB.$MYSQL_TABLE is failed";
echo $result
updateFlag
fi
#"-----------------append table-------------------"
if [ "$?" == "0" -a "$flag" == "0" ]; then
appendData
fi
elif [ "$?" != "0" -a "$flag" == "0" ]; then
flag=5;
TIME=`date +%H:%M:%S`
result="mysql -h $HOST -u$USER -p$PASSWD -e\" use $MYSQL_DB ; select * from $MYSQL_TABLE limit 1 \" is failed"
echo $result
updateFlag
fi
impala-shell --quiet -B -q " invalidate metadata $HIVE_DB.$HIVE_TABLE ";
}
host_conf='127.0.0.1'
user_conf='root'
passwd_conf='123456'
db_conf='bigdata_shell'
table_conf='sqoop_append'
sqoop_temp_dir='/user/root/temp_sqoop'
TIME=`date +%H:%M:%S`
startTime=$TIME
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e \"use $db_conf;select mysqldb,tablename,hivedb,host,port,user,password,sqoop_temp_dir,hdfs_dir,num_mappers,tuomindb from $table_conf where is_enabled=0 and pgid = $pgid ;\" |cat "
arr=`mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e "use $db_conf;select mysqldb,tablename,hivedb,host,port,user,password,sqoop_temp_dir,hdfs_dir,num_mappers,tuomindb from $table_conf where is_enabled=0 and pgid = $pgid ;" |cat `;
split=(${arr//,,,/ })
len=${#split[@]}
for (( i=0; i < $len-1; i=i+11 ))
do
HOST=${split[$i+3]}
PORT=${split[$i+4]}
USER=${split[$i+5]}
PASSWD=${split[$i+6]}
MYSQL_TABLE=${split[$i+1]}
MYSQL_DB=${split[$i]}
HIVE_DB=${split[$i+2]}
TEMPDIR=${split[$i+7]}
HDFS_DIR=${split[$i+8]}
NUMMAPPERS=${split[$i+9]}
TUOMIN_DB=${split[$i+10]}
echo "HOST="$HOST;
echo "PORT="$PORT;
echo "USER="$USER;
echo "PASSWD="$PASSWD;
echo "MYSQL_TABLE="$MYSQL_TABLE;
echo "MYSQL_DB="$MYSQL_DB;
echo "HIVE_DB="$HIVE_DB;
echo "TEMPDIR="$TEMPDIR;
echo "HDFS_DIR="$HDFS_DIR;
echo "NUMMAPPERS="$NUMMAPPERS;
echo "TUOMIN_DB="${TUOMIN_DB};
HIVE_TABLE=$MYSQL_TABLE
TEMPDIR=${TEMPDIR}/$HIVE_TABLE
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e \"use $db_conf;UPDATE $table_conf SET result='',status='running',updatetime='$DATE',start_time='$DATE $TIME' WHERE mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid ;\""
mysql -h $host_conf -u$user_conf -p$passwd_conf -N -e "use $db_conf;UPDATE $table_conf SET result='',status='running',updatetime='$DATE',start_time='$DATE $TIME' WHERE mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;"
if [ "$?" == "0" ]; then
echo "#mysql -h $host_conf -uroot -p$passwd_conf -N -e ""\"use $db_conf;select is_tuomin from $table_conf where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' limit 1 ;\"";
is_tuomin=`mysql -h $host_conf -uroot -p$passwd_conf -N -e "use $db_conf;select is_tuomin from $table_conf where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' limit 1 ;"`
if [ "$is_tuomin" == "0" ]; then
#-------------------- create tuomin datebases -------------------------
echo "impala-shell -q \" CREATE DATABASE IF NOT EXISTS ${TUOMIN_DB} \";";
impala-shell -q " CREATE DATABASE IF NOT EXISTS ${TUOMIN_DB}";
runSqoop
echo "#mysql -h $host_conf -uroot -p$passwd_conf -N -e \"use $db_conf;select tuominsql from $table_conf where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and is_tuomin = 0 limit 1 ;\"";
tuominsql=`mysql -h $host_conf -uroot -p$passwd_conf -N -e "use $db_conf;select tuominsql from $table_conf where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and is_tuomin = 0 limit 1 ;"`
echo "impala-shell --quiet -B -q \" invalidate metadata $HIVE_DB.$HIVE_TABLE \";"
impala-shell --quiet -B -q " invalidate metadata $HIVE_DB.$HIVE_TABLE ";
echo "#impala-shell -q \"DROP TABLE IF EXISTS ${TUOMIN_DB}.$MYSQL_TABLE;\"";
impala-shell -q "DROP TABLE IF EXISTS ${TUOMIN_DB}.$MYSQL_TABLE;"
echo "-----------------run tuominsql-------------------"
echo "#impala-shell -q \"$tuominsql\"";
impala-shell -q "$tuominsql"
if [ "$?" == "0" ] ;then
echo "impala-shell -q \"DROP TABLE IF EXISTS ${HIVE_DB}.$MYSQL_TABLE;\";"
# impala-shell -q "DROP TABLE IF EXISTS ${HIVE_DB}.$MYSQL_TABLE;"
else
TIME=`date +%H:%M:%S`
flag=7;
result="$DATE $TIME $HIVE_TABLE tuominsql is failed"
echo $result
updateFlag
fi
elif [ "$is_tuomin" == "1" ]; then
#----------dev,prod-------------
# HDFS_DIR=${HDFS_DIR//$HIVE_DB/$TUOMIN_DB}
# TEMPDIR=${TEMPDIR//$HIVE_DB/$TUOMIN_DB}
# HIVE_DB=${TUOMIN_DB}
runSqoop
fi
if [ "$?" != "0" ]; then
TIME=`date +%H:%M:%S`
flag=6;
result="$DATE $TIME $HIVE_TABLE tuomin is failed"
echo $result
updateFlag
fi
hive_num=-1
# HIVE_DB=${TUOMIN_DB}
insertNum
if [ "$?" == "0" -a "$flag" == "0" ]; then
TIME=`date +%H:%M:%S`
result="$HIVE_DB.$HIVE_TABLE is successful"
echo $result
updateFlag
fi
else
flag=5;
TIME=`date +%H:%M:%S`
result="mysql -h $HOST -u$USER -p$PASSWD -e\" use $MYSQL_DB ; UPDATE is failed"
echo $result
updateFlag
fi
TIME=`date +%H:%M:%S`
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -e \"use $db_conf; update $table_conf set end_time='$DATE $TIME',resultcode='$flag',status='end',run_time=TIMEDIFF(end_time,start_time) where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;\";";
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set end_time='$DATE $TIME',resultcode='$flag',status='end',run_time=TIMEDIFF(end_time,start_time) where mysqldb='$MYSQL_DB' and tablename='$MYSQL_TABLE' and pgid=$pgid;"
done
if [ "$?" == "0" ] ;then
echo "-----------------check data-------------------"
pid=`ps -ef |grep "$0 ${param1} 1000" |grep -v grep |awk '{print $2}'`
pids=(${pid//,,,/ })
pidl=${#pids[@]}
echo $pidl
if [ $pidl -gt 0 ]; then
exit 0;
else
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -e \"use $db_conf; select id,pgid from $table_conf where mysqldb='$MYSQL_DB' and is_enabled=0 and status='end' and resultcode != 0; \" |cat ;";
errorpgid=`mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; select id,pgid from $table_conf where mysqldb='$MYSQL_DB' and is_enabled=0 and status='end' and resultcode != 0;" |cat`
errorpgids=(${errorpgid//,,,/ })
errorpgidl=${#errorpgids[@]}
if [ $errorpgidl > 1 ]; then
echo "#mysql -h $host_conf -u$user_conf -p$passwd_conf -e \"use $db_conf; update $table_conf set pgid=1000 where mysqldb='$MYSQL_DB' and is_enabled=0 and resultcode != 0; \" ;";
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set pgid=1000 where mysqldb='$MYSQL_DB' and is_enabled=0 and resultcode != 0;"
echo "sh $0 ${param1} 1000"
sh $0 ${param1} 1000
for (( i=0; i < $errorpgidl-1; i=i+2 ))
do
ID=${errorpgids[$i]}
PGID=${errorpgids[$i+1]}
mysql -h $host_conf -u$user_conf -p$passwd_conf -e "use $db_conf; update $table_conf set pgid=$PGID where id=$ID ;"
done
fi
fi
echo "-----------------END-------------------"
fi
if [ "$?" == "0" -a "zhifu" == "${param1}" ] ;then
zhifupid=`ps -ef |grep "$0 ${param1}" |grep -v grep |awk '{print $2}'`
pidsplit=(${zhifupid//,,,/ })
pidlen=${#pidsplit[@]}
if [ 3 -eq $pidlen ]; then
processid=`yarn application -list|grep 'com.yanxin.javaspark.application.NewImportHDFSHiveData'|awk '{print $1}'`
if [ $? == 0 ]; then
echo "applicationId="$processid
yarn application -kill $processid
hdfs dfs -rm -R /user/hive/warehouse/real_time_zhifu.db/*
hdfs dfs -cp /user/hive/warehouse/transfer_zhifu.db/* /user/hive/warehouse/real_time_zhifu.db
sh /root/harry/NewImportHDFSHiveData_submit.sh
fi
else
echo "#the zhifupid is $zhifuid";
exit 0;
fi
fi
if [ "$?" == "0" -a "sjdk" == "${param1}" ] ;then
sjdkpid=`ps -ef |grep "$0 ${param1}" |grep -v grep |awk '{print $2}'`
pidsplit=(${sjdkpid//,,,/ })
pidlen=${#pidsplit[@]}
echo $pidlen
if [ 3 -eq $pidlen ]; then
DATE=`date +%Y-%m-%d`
TIME=`date +%H:%M:%S`
YESTERDAY=`date -d "1 day ago" +%Y-%m-%d`
sql="insert into data_market.last_process_time SELECT 'NULL' AS syncid,AA.hivedb,AA.tablename,'0' as status,'$DATE $TIME' as synctime , AA.hive_count_num,'sjdk' mysqldb,'$YESTERDAY' datatime from
(SELECT hivedb,tablename,hive_count_num FROM data_market.last_process_time WHERE mysqldb='sjdk' and substr(synctime,1,10)='$YESTERDAY' AND tablename NOT IN
(SELECT tablename from data_market.last_process_time WHERE mysqldb='sjdk' and substr(synctime,1,10)='$DATE')) as AA;"
echo "impala-shell -q \"$sql\";"
impala-shell -q "$sql";
else
echo "#the sjdkpid is $sjdkpid";
exit 0;
fi
fi