hive常用参数优化

--中间结果压缩
set hive.exec.compress.intermediate=true;
set mapred.map.output.compression.codec=org.apache.hadoop.io.compress.SnappyCodec;
--sequeuencefile文件输出结果压缩
set hive.exec.compress.output=true;
set io.seqfile.compression.type=block;
set mapred.output.compress=true; 
set mapred.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;

--输入数据合并小文件
set mapred.max.split.size=256000000;
set mapred.min.split.size.per.node=100000000;
set mapred.min.split.size.per.rack=100000000;
set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; 

--输出数据合并小文件
set hive.merge.mapfiles=true;
set hive.merge.mapredfiles=true;
set hive.merge.size.per.task=256000000;
set hive.merge.smallfiles.avgsize=16000000;

--并行执行
set hive.exec.parallel=true;
--加大reduce处理数据量
set hive.exec.reducers.bytes.per.reducer=256000000;

--lzo文本文件压缩
set hive.exec.compress.output=true
set mapreduce.output.fileoutputformat.compress=true
set mapreduce.output.fileoutputformat.compress.codec=com.hadoop.compression.lzo.LzoCodec

--分析统计hive
analyze table hello compute statistics for columns;
describe formatted hello.sex;
describe formatted hello.age;