Sqoop将MySQL数据导入到hive中样例

#!/bin/bash

#program date parameter
yesterday=`date --date="${1} -1 day" +%Y-%m-%d`
V_MYSQL_DT=`date --date="${1} -1 day" +%Y%m`

HADOOP_HOME=/usr/hdp/2.6.5.0-292/hadoop
SQOOP_HOME=/usr/hdp/2.6.5.0-292/sqoop

begin="$yesterday 00:00:00"
end="$yesterday 02:00:00"
separator=','

#sqoop任务运行mapreduce任务数量
task=1
#源数据表对应的hive表
hive_table_name="signaling.signaling_user_login_logout"
username="root"
password="root"
database="bigdata"
connstr="jdbc:mysql://localhost.com:3306"
table_name="signaling_user_login_logout_$V_MYSQL_DT"
target_table=signaling_user_login_logout
datadir="hdfs://mulu/signaling.db/$target_table/$yesterday"
#分区字段
key="day"

$HADOOP_HOME/bin/hadoop fs -rm -r -f $datadir
hive -e "set mapreduce.job.queuename=default;alter table signaling.$target_table drop partition($key='$yesterday');"

#sqoop命令导入
$SQOOP_HOME/bin/sqoop import \
-D mapred.job.queue.name=default \
--connect $connstr/$database \
--username $username \
--password $password \
--fields-terminated-by $separator \
--hive-import \
--query "SELECT * FROM $table_name WHERE action_time >= '$begin' AND action_time <= '$end' AND  \$CONDITIONS"  \
--hive-table $hive_table_name \
--hive-partition-key $key \
--hive-partition-value $yesterday \
--target-dir $datadir \
--null-string '\\N' \
--null-non-string '\\N' \
--hive-drop-import-delims \
-m 1