hive 小文件数据合并

hive 数据有时候需要进行数据合并

 

#!/bin/bash

hadoop jar /software/servers/bdp_tools/mergefiles-1.7.jar -u lzo -p hdfs://ns1/user/dd_edw/adm.db/table_name/dt=2018-09-01

2018/12/06 今儿谈话聊天更精进一部lovely

 

#!/bin/bash

end_date='2018-12-05'
beg_date='2017-01-01'


beg_s=`date -d "$beg_date" +%s`
end_s=`date -d "$end_date" +%s`

while [ $beg_s -le $end_s ];do

	cur_day=`date -d @$beg_s +"%Y-%m-%d"`;
	
	echo $cur_day
	hadoop jar /software/servers/bdp_tools/mergefiles-1.7.jar -u orc -p hdfs://ns1/user/dd_edw/adm.db/table_name/dt=$cur_day/tp='day'
    hadoop jar /software/servers/bdp_tools/mergefiles-1.7.jar -u orc -p hdfs://ns1/user/dd_edw/adm.db/table_name/dt=$cur_day/tp='week'
	
	beg_s=$((beg_s+86400));
	
done 
echo 'process merger done'