hive命令学习

Hive学习
默认日志地址 /tmp/root(用户名)/hive.log
hive -hiveconf hive.root.logger=DEBUG,console   //重启调试。
HIVE 基本操作 hive shell 
新建表
line_name,deal_yymmdd,deal_hh,count
线路10,20150101,06,714
线路10,20150101,07,2008
线路10,20150101,08,2652
线路10,20150101,09,2691
创建内部表
CREATE TABLE bus (line_name string,deal_yymmdd string, deal_hh string, count int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ; 
CREATE TABLE bus2 (line_name string,deal_yymmdd string, deal_hh string, count int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' COLLECTION ITEMS TERMINATED BY ','; 
创建外部表  对于使用external建表完成后，再drop掉表，表中的数据还在文件系统中,需要LOCATION指定存放数据的文件路径(如果目录存在会删除目录下文件重新建立)
COMMENT描述这张表
CREATE EXTERNAL TABLE IF NOT EXISTS bus (
line_name string,
deal_yymmdd string, 
deal_hh string, 
count int) 
COMMENT 'This is the staging page view table'   
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
LOCATION  '/opt/bus2'; 
#导入数据reduces2.txt到bus表 hdfs数据移动到/user/hive/warehouse/bus OVERWRITE 关键字，则目标表（或者分区）中的内容（如果有）会被删除
导入本地文件
LOAD DATA LOCAL INPATH '/home/reduces2.txt' OVERWRITE INTO TABLE bus;
导入HDFS文件
LOAD DATA INPATH '/opt/bus/part-m-00000' OVERWRITE INTO TABLE bus;
查看表和数据
show tables;
#正则匹配表名
show tables '*b*';
#查看表数据
select * from bus;
#查看表结构
desc bus; 
DESCRIBE bus;
#增加一个字段
ALTER TABLE bus ADD COLUMNS (week String);
#重命令表名
ALTER TABLE bus2 RENAME TO bus3;
删除表
DROP TABLE bus3;
从其他表导入数据
Specified key was too long; max key length is 767 bytes出错解决：一般是由于mysql的最大索引长度导致1、数据库的字符集为latin1；2、set global binlog_format='MIXED'
CREATE TABLE bus4 (line_name string,deal_yymmdd string, deal_hh string, count int,week string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',';
INSERT OVERWRITE TABLE bus4 SELECT * FROM bus ;
DROP TABLE bus4;
创建表并从其他表导入数据
CREATE TABLE bus4 AS SELECT * FROM bus;
仅复制表结构不导数据
CREATE TABLE bus4 LIKE bus;
数据导出   杀死JAVA进程kill -9 11110(pid)
导出到HDFS上其他目录
hadoop fs -cp /user/hive/warehouse/bus  /opt
通过Hive导出到本地文件系统
INSERT OVERWRITE LOCAL DIRECTORY '/opt/tools/' SELECT * FROM bus;
Hive查询HiveQL
普通查询：排序，列别名，嵌套子查询 ,列出前多少条
FROM (
SELECT deal_yymmdd as c2,count FROM bus   
) t
SELECT t.c2, t.count
WHERE count>3000  
LIMIT 2;  
排序ORDER BY ASC(升序),DESC(降序)||SORT BY   ASC(升序),DESC(降序)顺序将根据列类型而定。如果数字类型的列，则排序顺序也以数字顺序。如果字符串类型的列，则排序顺序将字典顺序,只能在单机范围内排序.
select line_name,deal_yymmdd,deal_hh,count from bus1 ORDER BY deal_yymmdd ASC,deal_hh DESC;
连接查询：JOIN
SELECT t1.deal_yymmdd,t1.deal_hh,t2.deal_yymmdd,t2.deal_hh
FROM bus t1 JOIN bus4 t2 on t1.deal_yymmdd=t2.deal_yymmdd
WHERE t1.count>3000
LIMIT 5;
聚合查询1：count, avg 总数 平均值
SELECT count(*), avg(count) FROM bus WHERE deal_yymmdd='20150104';
聚合查询2：count, distinct  总数 不重复的值
SELECT count(DISTINCT line_name) FROM bus;
聚合查询3：GROUP BY, HAVING  where在分组之前过滤数据having在分组之后过滤数据 ORDER BY ASC(升序),DESC(降序);
SELECT avg(count),sum(count) FROM bus GROUP BY line_name;
SELECT avg(count),deal_yymmdd,sum(count) FROM bus GROUP BY deal_yymmdd,line_name HAVING sum(count)>40000;
Hive视图 Hive视图和数据库视图的概念是一样的
CREATE VIEW v_bus AS SELECT deal_hh,count FROM bus where deal_yymmdd='20150104';
select * from v_bus;
删除视图
DROP VIEW IF EXISTS v_bus;
Hive分区表  内部表以line_name 分区
CREATE TABLE bus1(
line_name string,
deal_yymmdd string, 
deal_hh string, 
count int,
week string) 
PARTITIONED BY (line string)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ',';
装载
LOAD DATA  INPATH '/opt/bus/part-m-00001' OVERWRITE INTO TABLE bus1 PARTITION (line='10');
LOAD DATA  INPATH '/opt/bus/part-m-00000' OVERWRITE INTO TABLE bus1 PARTITION (line='15');
外部表建分区 可以不指定存储地址
CREATE EXTERNAL TABLE IF NOT EXISTS bus2(
line_name string,
deal_yymmdd string, 
deal_hh string, 
count int) 
PARTITIONED BY (yy string,mm string)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ',';
增加外部分区
ALTER TABLE  bus2 ADD PARTITION(yy='2015',mm='02') location '/opt/bus/2015/02';
ALTER TABLE  bus2 ADD 
PARTITION(yy='2015',mm='01') location '/opt/bus/2015/01'
PARTITION(yy='2015',mm='02') location '/opt/bus/2015/02';  --同时增加多个分区
删除分区
ALTER TABLE bus2 DROP PARTITION (yy='2015',mm='02');
装载
LOAD DATA  INPATH '/opt/bus/part-m-00001' OVERWRITE INTO TABLE bus2 PARTITION (yy='2015',mm='02');
查看分区表
SHOW PARTITIONS bus1;
查询数据
select * from bus2 where yy='2015' and mm='02';
select * from bus1 where deal_yymmdd='20150104';
select * from bus1 where line='15' and deal_yymmdd='20150104';
表重命名 数据所在的位置和分区名并不改变
ALTER TABLE bus2 RENAME TO bus3;
改变列名、数据类型、注释、列位置或者它们的任意组合
ALTER TABLE table_name CHANGE [COLUMN] col_old_name col_new_name column_type
[COMMENT col_comment] [FIRST|AFTER column_name]
通过查询语句向表中插入数据  --如何没有指定分区位置默认/user/hive/warehouse/bus2/yy=2015/mm=01
INSERT OVERWRITE TABLE bus2 
PARTITION (yy='2015',mm='01')
select line_name,deal_yymmdd,deal_hh,count from bus2 where yy='2015' and mm='02';
导出数据 
INSERT OVERWRITE DIRECTORY '/opt/bus2' 
select line_name,deal_yymmdd,deal_hh,count from bus2 where yy='2015' and mm='03'
分成多个reduce的方法DISTRIBUTE BY
set mapred.reduce.tasks=2;---可以规定reduce个数，也可以不规定
Select sale_id, amount from t_order
Distribute by sale_id
Sort by sale_id, amount;


Hive交互式模式
    quit,exit:  退出交互式shell
    reset: 重置配置为默认值
    set <key>=<value> : 修改特定变量的值(如果变量名拼写错误，不会报错)
    set :  输出用户覆盖的hive配置变量
    set -v : 输出所有Hadoop和Hive的配置变量
    add FILE[S] *, add JAR[S] *, add ARCHIVE[S] * : 添加 一个或多个 file, jar, archives到分布式缓存
    list FILE[S], list JAR[S], list ARCHIVE[S] : 输出已经添加到分布式缓存的资源。
    list FILE[S] *, list JAR[S] *,list ARCHIVE[S] * : 检查给定的资源是否添加到分布式缓存
    delete FILE[S] *,delete JAR[S] *,delete ARCHIVE[S] * : 从分布式缓存删除指定的资源
    ! <command> :  从Hive shell执行一个shell命令
    dfs <dfs command> :  从Hive shell执行一个dfs命令
    <query string> : 执行一个Hive 查询，然后输出结果到标准输出
    source FILE <filepath>:  在CLI里执行一个hive脚本文件
bin/hive 是一个 shell 工具，它可以用来运行于交互或批处理方式配置单元查询
Usage: hive [-hiveconf x=y]* [<-i filename>]* [<-f filename>|<-e query-string>] [-S]
-i <filename> Initialization Sql from file (executed automatically and silently
before any other commands)
-e 'quoted query string' Sql from command line
-f <filename> Sql from file
-S Silent mode in interactive shell where only data is emitted
-hiveconf x=y Use this to set hive/hadoop configuration variables.
-e and -f cannot be specified together. In the absence of these options, interactive shell is
started.
However, -i can be used with any other options. Multiple instances of -i can be used to
execute multiple init scripts.
To see this usage help, run hive -h
运行一个查询
hive -e 'select count(*) from bus1'
将查询结果导出到一个文件
hive -S -e 'select count(*) from bus1' > a.txt
运行一个脚本
hive -f /home/my/hive-script.sql