两种方式用Shell实现把Hive表数据导入到redis

话不多说，直接上干货，各位改下目录就能直接使用，希望能点赞收藏。

脚本如下所示：

#!/bin/bash


#expect -c "
#    spawn scp /root/liuyuqiang.txt [email protected]:/root/
#    expect {
#        \"*assword\" {set timeout 20; send \"gbase\r\"; exp_continue;} 
#    }
#expect eof"

pwd=/root
day=`date +%Y%m%d`
before_day=`date -d -1day +%Y%m%d`

if [ -d $pwd/$day ]
then
	echo "`date '+%Y-%m-%d %H:%M:%S'`----$pwd/$day 目录存在，不需要创建" > $pwd/load_redis_$day.log
else
	mkdir $pwd/$day/
	echo "`date '+%Y-%m-%d %H:%M:%S'`----$pwd/$day 目录不存在，创建目录成功" > $pwd/load_redis_$day.log
fi



function export_hive(){
echo "导出数据为 insert overwrite local directory '$pwd/$day/' row format delimited fields terminated by '|' select a,b,c,d from dw_wangxinqing_yyyymmdd1 where day_id=$day;" >> $pwd/load_redis_$day.log
echo "`date '+%Y-%m-%d %H:%M:%S'` -------开始从hive导出数据--------"  >> $pwd/load_redis_$day.log
hive -e "insert overwrite local directory '$pwd/$day/' row format delimited fields terminated by '|' select a,b,c,d from dw_wangxinqing_yyyymmdd1 where day_id=$before_day;"
mv $pwd/$day/000000* $pwd/$day/hive_$before_day.txt

if [ $? -ne 0 ]
then
	echo "`date '+%Y-%m-%d %H:%M:%S'`-----导出失败-----" >> $pwd/load_redis_$day.log
	exit
else
	echo "`date '+%Y-%m-%d %H:%M:%S'`-----导出成功-----" >> $pwd/load_redis_$day.log
fi

}

function Clear_hive(){

rm -f $pwd/$day/redis_$before_day.txt

while read row
do
	telnum=$(echo "${row}" |awk -F "|" '{print $1}')
	region=$(echo "${row}" |awk -F "|" '{print $2}')
	crowd=$(echo "${row}" |awk -F "|" '{print $3}')
	activeType=$(echo "${row}" |awk -F "|" '{print $4}')
	
	if [ ! -n "$telnum" ]
	then
		telnum='" "'
	fi
	
	if [ ! -n "$region" ]
	then
		region='" "'
	fi
	
	if [ ! -n "$crowd" ]
	then
		crowd='" "'
	fi
	
	if [ ! -n "$activeType" ]
	then
		activeType='" "'
	fi
	
	echo "HMSET asiainfo:${telnum} telnum ${telnum} region ${region} crowd ${crowd} activeType ${activeType}" >> $pwd/$day/redis_$before_day.txt
	echo "EXPIRE asiainfo:${telnum} 300" >> $pwd/$day/redis_$before_day.txt
done < $pwd/$day/hive_$before_day.txt
	unix2dos $pwd/$day/hive_$before_day.txt
	echo "`date '+%Y-%m-%d %H:%M:%S'`-----格式转换成功-----" >> $pwd/load_redis_$day.log
	scp $pwd/$day/redis_$before_day.txt hadoop1:$pwd/
	if [ $? -ne 0 ]
	then
		echo "`date '+%Y-%m-%d %H:%M:%S'`-----传输到目标服务器失败-----" >> $pwd/load_redis_$day.log
	exit
	else
		echo "`date '+%Y-%m-%d %H:%M:%S'`-----传输到目标服务器成功-----" >> $pwd/load_redis_$day.log
fi
}


function Load_redis(){
	echo "-------------可以load到redis------------" >> $pwd/load_redis_$day.log
	ssh hadoop1 "time cat $pwd/redis_$before_day.txt | /usr/bin/redis-cli --pipe -a 'Asdf12345!'" >> $pwd/load_redis_$day.log
}


function start(){
export_hive
Clear_hive
Load_redis
}

start

效果如下：

第二种：
第一种方法用循环处理的话，非常的慢，200万行数据要花费7个小时，所以就用hive进行了数据处理，5分钟不到就搞定了。
因为在云桌面里面，无法复制出来，只有看图片了。



hive.sql：