Hive脚本、Hive API、Hive 自定义函数

1、脚本

./hive -e 'select * from t_student';

hive -e "hql"
编写hive脚本
vi weblog.sh
#!/bin/bash
day_str="2017-09-17"
#day_str=`date +"%Y-%m-%d"`

hive_exec=/opt/apache-hive-1.2.2-bin/bin/hive

HQL_user_active_day="insert into weblog.t_activity_user partition(day='$day_str') select tmp.ip ip,tmp.uname uname,tmp.access_time access_time,tmp.url url from (select ip,uname,access_time,url,row_number() over(partition by uname order by access_time) rk from weblog.t_web_log where day='$day_str') tmp where rk = 1"

HQL_user_new_day="insert into weblog.t_new_user partition(day='$day_str') select tau.ip,tau.uname,tau.access_time,tau.url from weblog.t_activity_user tau left join weblog.t_history_user thu on tau.uname = thu.uname where tau.day = '$day_str' and thu.uname is null"

HQL_user_history_day="insert into weblog.t_history_user select uname from weblog.t_new_user where day = '$day_str'"

$hive_exec -e "$HQL_user_active_day"
$hive_exec -e "$HQL_user_new_day"
$hive_exec -e "$HQL_user_history_day"

还可以把hql语句写在一个sql脚本里
用hive -f sqlscript.sql

2、JAVA-API

    org.apache.hive
    hive-jdbc
    1.2.2
    provided

package com.bawei.hive;

import java.sql.*;

public class ClientDemo {
    public static void main(String[] args) throws ClassNotFoundException, SQLException {
        Class.forName("org.apache.hive.jdbc.HiveDriver");
        Connection conn = DriverManager.getConnection("jdbc:hive2://hdp1:10000", "root", "");

        Statement stat = conn.createStatement();

        stat.execute("load data local inpath '/root/09-17.log' into table weblog.t_web_log partition(day='09-17')");

        stat.execute("use weblog");

        stat.execute("insert into t_activity_user partition(day='09-17')\n" +
                "select\n" +
                "tmp.ip ip,tmp.uname uname,tmp.access_time access_time,tmp.url url\n" +
                "from\n" +
                "(select\n" +
                "ip,uname,access_time,url,row_number() over(partition by uname order by access_time) rk\n" +
                "from\n" +
                "t_web_log\n" +
                "where day='09-17') tmp\n" +
                "where rk = 1");

        stat.execute("insert into t_new_user partition(day='09-17')\n" +
                "select\n" +
                "tau.ip,tau.uname,tau.access_time,tau.url\n" +
                "from\n" +
                "t_activity_user tau left join t_history_user thu on tau.uname = thu.uname\n" +
                "where tau.day = '09-17' and thu.uname is null");

        stat.execute("insert into t_history_user\n" +
                "select uname from t_new_user where day = '09-17'");

        ResultSet rs = stat.executeQuery("select * from t_activity_user where day = '09-17'");

        System.out.println("ip      |      uname     |     access_time     |      url");
        while (rs.next()){
            String ip = rs.getString(1);
            String uname = rs.getString(2);
            String accessTime = rs.getString(3);
            String url = rs.getString(4);

            System.out.println(ip + " | " + uname + " | " + accessTime + " | " + url);
        }

        System.out.println("");
        System.out.println("=============================================================");
        System.out.println("");

        ResultSet rs2 = stat.executeQuery("select * from t_new_user where day = '09-17'");
        System.out.println("ip      |      uname     |     access_time     |      url");
        while (rs2.next()){
            String ip = rs2.getString(1);
            String uname = rs2.getString(2);
            String accessTime = rs2.getString(3);
            String url = rs2.getString(4);

            System.out.println(ip + " | " + uname + " | " + accessTime + " | " + url);
        }

        System.out.println("");
        System.out.println("=============================================================");
        System.out.println("");

        ResultSet rs3 = stat.executeQuery("select * from t_history_user");
        System.out.println("uname");
        while (rs3.next()){
            System.out.println(rs3.getString(1));
        }

    }
}

3、Hive自定义函数

package com.bawei.hive;

import com.alibaba.fastjson.JSONObject;
import org.apache.hadoop.hive.ql.exec.UDF;

public class JsonToString extends UDF {

    public String evaluate(String json,String param){
        JSONObject jsonObject = JSONObject.parseObject(json);
        return jsonObject.getString(param);
    }
}

pom.xml

    4.0.0

    com.bawei
    hive
    1.0-SNAPSHOT
    
        
        
        
        

        
        
            org.apache.hive
            hive-exec
            1.2.2
            provided
        

        
        
            com.alibaba
            fastjson
            1.2.47
        

    
    
        
            
                org.apache.maven.plugins
                maven-shade-plugin
                2.4.3
                
                    
                        package
                        
                            shade
                        
                        
                            
                                
                                    *:*
                                    
                                        META-INF/*.SF
                                        META-INF/*.DSA
                                        META-INF/*.RSA
                                    
                                
                            
                        
                    

                
            
        
    

1.test范围是指测试范围有效,在编译和打包时都不会使用这个依赖
2.compile范围是指编译范围内有效,在编译和打包时都会将依赖存储进去
3.provided依赖,在编译和测试过程中有效,最后生成的war包时不会加入 例如:
servlet-api,因为servlet-api tomcat服务器已经存在了,如果再打包会冲突
4.runtime在运行时候依赖,在编译时候不依赖
默认依赖范围是compile

创建永久函数

方法一：

add jar /opt/apache-hive-1.2.2-bin/lib/hive-udf2.jar; 
create function getjson AS 'com.bawei.hive.JsonToString';

方法二：
在hive-site.xml里添加jar包

    hive.aux.jars.path
    file:///opt/apache-hive-1.2.2-bin/lib/hive-udf2.jar

create function getjson AS 'com.bawei.hive.JsonToString';

方法三：

create function getjson as 'com.bawei.hive.JsonToString' using jar 'hdfs://hiveudf/hive-udf2.jar';

drop function getjson

创建临时函数：temporary

create temporary function ……