Hive内置函数的应用案例

1：计算两个时间相差多少天：hive内置函数datediff
 例子Demo：datediff('2009-07-28','2009-07-30')=2
2:也可以写自定义函数
package cdel.edu.hive.udf.demo04;


import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;


import org.apache.hadoop.hive.ql.exec.UDF;


/**
 * 计算一个时间到今天的相差多少天
 * @author dell
 *
 */
public class MyDateDiff extends UDF {


public String evaluate(String str) {
try {
String  today = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date().getTime());
int days = daysBetween(str,today);
return String.valueOf(days);
} catch (Exception e) {
return null;
}
}


/**
* 计算两个日期之间相差的天数
*
* @param smdate
*            较小的时间
* @param bdate
*            较大的时间
* @return 相差天数
* @throws ParseException
*/
public static int daysBetween(Date smdate, Date bdate)
throws ParseException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
smdate = sdf.parse(sdf.format(smdate));
bdate = sdf.parse(sdf.format(bdate));
Calendar cal = Calendar.getInstance();
cal.setTime(smdate);
long time1 = cal.getTimeInMillis();
cal.setTime(bdate);
long time2 = cal.getTimeInMillis();
long between_days = (time2 - time1) / (1000 * 3600 * 24);


return Integer.parseInt(String.valueOf(between_days));
}


/**
* 字符串的日期格式的计算
*/
public static int daysBetween(String smdate, String bdate)
throws ParseException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
Calendar cal = Calendar.getInstance();
cal.setTime(sdf.parse(smdate));
long time1 = cal.getTimeInMillis();
cal.setTime(sdf.parse(bdate));
long time2 = cal.getTimeInMillis();
long between_days = (time2 - time1) / (1000 * 3600 * 24);


return Integer.parseInt(String.valueOf(between_days));
}


public static void main(String[] args) throws Exception {
   String str="2015-04-27";
       System.out.println(new MyDateDiff().evaluate(str));
}
}




-创建自定义函数
add jar /home/admin/xxg/MyDateDiff.jar;
create temporary function myDateDiff as 'cdel.edu.hive.udf.demo04.MyDateDiff';


--hive 统计 
 
-- 最近30 天内学员的做题量统计 
--分网统计
create table  qz_count as
SELECT   mps.userID,mem.memberid,COUNT(distinct qpq.questionID) as  doQuesCount,mem.domain,
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))   as daytime
from   tmp_qz_member_paper_score  mps   
left join  tmp_qz_member_paper_question qpq  on  qpq.paperScoreID = mps.paperScoreID  and mps.domain = qpq.domain 
left join order.tmp_member_info_new  mem on mem.uid = mps.userID and mem.domain = mps.domain
where   cast(myDateDiff(concat_ws(' ',substr(mps.createTime,1,10),'')) as int) <=30
group  by  mps.userID,mem.memberid ,mem.domain,
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))
having  COUNT(distinct qpq.questionID)  >0
order by   doQuesCount  desc  ;


SELECT   mps.userID,mem.memberid,COUNT(distinct qpq.questionID) as  doQuesCount,mem.domain,
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))   as daytime
from   tmp_qz_member_paper_score  mps   
left join  tmp_qz_member_paper_question qpq  on  qpq.paperScoreID = mps.paperScoreID  and mps.domain = qpq.domain 
left join order.tmp_member_info_new  mem on mem.uid = mps.userID and mem.domain = mps.domain
where  mps.userID='48888922'   and  cast(myDateDiff(concat_ws(' ',substr(mps.createTime,1,10),'')) as int) <=31
group  by  mps.userID,mem.memberid ,mem.domain,
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))
having  COUNT(distinct qpq.questionID)  >0
order by   doQuesCount  desc  ;


SELECT   qpq.*
from   tmp_qz_member_paper_score  mps   
left join  tmp_qz_member_paper_question qpq  on  qpq.paperScoreID = mps.paperScoreID  and mps.domain = qpq.domain 
where  mps.userID='48888922'   limit 100;


经验验证：没有医学网做题题目信息
select  distinct logdata from tmp_qz_member_paper_question where domain='med' limit 10
验证数据
select * from qz_count  where userid=48888922 order by   doQuesCount  desc limit 100;


select * from qz_count  where doQuesCount>=500 order by   doQuesCount  desc ;


-- 不分网统计
create table  qz_count as
SELECT   mps.userID,mem.memberid,COUNT(distinct qpq.questionID) as  doQuesCount,
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))   as daytime
from   tmp_qz_member_paper_score  mps   
left join  tmp_qz_member_paper_question qpq  on  qpq.paperScoreID = mps.paperScoreID  and mps.domain = qpq.domain 
left join order.tmp_member_info_new  mem on mem.uid = mps.userID and mem.domain = mps.domain
where   cast(myDateDiff(concat_ws(' ',substr(mps.createTime,1,10),'')) as int) <=30
group  by  mps.userID,mem.memberid , 
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))
having  COUNT(distinct qpq.questionID)  >0
order by   doQuesCount  desc  ;




-- 统计学员每天做多少套试卷
-- 分网统计
create table  paper_count as
select   mps.userID,mem.Memberid,mem.domain, COUNT(distinct mps.paperViewID) AS doPaperCount ,  
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))   as daytime
from tmp_qz_member_paper_score mps
left join  order.tmp_member_info_new  mem  on mem.uID = mps.userID  and mem.domain = mps.domain
where  cast(myDateDiff(concat_ws(' ',substr(mps.createTime,1,10),'')) as int) <=31
group  by  mps.userID,mem.Memberid, mem.domain,
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))
having  COUNT(distinct mps.paperViewID) >0
order by   doPaperCount   desc;


 -- 不分网统计
select   mps.userID,mem.Memberid,COUNT(distinct mps.paperViewID) AS doPaperCount ,  
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))   as daytime
from tmp_qz_member_paper_score mps
left join  order.tmp_member_info_new  mem  on mem.uID = mps.userID  and mem.domain = mps.domain
where   mps.userID=48888922
group  by mps.userID,mem.Memberid,
to_date(concat_ws(' ',substr(mps.createTime,1,10),substr(mps.createTime,11,length(mps.createTime))))
having  COUNT(distinct mps.paperViewID) >0
order by   doPaperCount   desc;




测试：
select * from  paper_count  order by   doPaperCount   desc  limit 100;


select   mps.userID,  mps.paperViewID  from tmp_qz_member_paper_score mps  where   mps.userID=48888922;






-- 导出本地文件中
insert overwrite local directory '/home/admin/xxg/qz/med/'
row format delimited
fields terminated by '\t'
select  *  from qz_count;






-- 最近30天内学员的提问数量
-- 针对主题的提问统计
 
SELECT ft.userID,ft.userName ,  COUNT(ft.userID)  as num FROM  FAQ_TOPIC  ft
group by FT.userid ,ft.userName




SELECT ft.userID,ft.userName ,  COUNT(af.USERID)  as num 
FROM  FAQ_TOPIC  ft
inner JOIN   faq   af on ft.topicID = af.topicID and af.istopic = 1
group by ft.userID,ft.userName


create table  faq_count as
SELECT   ft.userID, mem.memberid, COUNT(distinct ft.topicID) as num, mem.domain,
to_date(concat_ws(' ',substr(ft.createTime,1,10),substr(ft.createTime,11,length(ft.createTime))))   as daytime
FROM  tmp_faq_topic   ft  
left join order.tmp_member_info_new  mem on mem.uid = ft.userID   and mem.domain = ft.domain   
where     cast(myDateDiff(concat_ws(' ',substr(ft.createTime,1,10),'')) as int) <=30
group by ft.userID, mem.memberid,mem.domain,
to_date(concat_ws(' ',substr(ft.createTime,1,10),substr(ft.createTime,11,length(ft.createTime))))
having  COUNT(distinct ft.topicID)  >0
order by num desc  ;




--针对主题和继续提问的 --- 就是如果一个主题，两个继续的话 你统计的sum =3


create table  faq_count2 as
SELECT   ft.userID, mem.memberid, COUNT(distinct af.faqid) as num, mem.domain,
to_date(concat_ws(' ',substr(af.createTime,1,10),substr(af.createTime,11,length(af.createTime))))   as daytime
FROM   tmp_faq af  
left join  tmp_faq_topic   ft   on ft.topicID = af.topicID  and af.domain = ft.domain  
left join order.tmp_member_info_new  mem on mem.uid = ft.userID   and mem.domain = af.domain   
where     cast(myDateDiff(concat_ws(' ',substr(af.createTime,1,10),'')) as int) <=30
group by ft.userID, mem.memberid, mem.domain,
to_date(concat_ws(' ',substr(af.createTime,1,10),substr(af.createTime,11,length(af.createTime))))
having  COUNT(distinct af.faqid)  >0
order by num desc   limit 10;




-- 导出本地文件中
insert overwrite local directory '/home/admin/xxg/faq/'
row format delimited
fields terminated by '\t'
select  *  from faq_count;


-- 删除表信息
drop table qz_count;
drop table faq_count;
drop table paper_count;


-- 统计 
每天做题 >=3000道以上学员的个数
每天做题 >=2500  and <3000 道以上的学员个数
每天做题 >=2000  and <2500 道以上的学员个数
每天做题 >=1500 and <2000 道以上的学员个数
每天做题 >=1000 and <1500 道以上的学员个数
每天做题 >=500 and <1000 道以上的学员个数
每天做题 >=100 and <500 道以上的学员个数
每天做题 >=50 and <100 道以上的学员个数
每天做题 >0   and <50  道以上的学员个数


-- 根据学员分组统计，学员做题数量统计
select userid,memberid,domain,
SUM(case when doquescount=0    then 1 else 0 end) as n0,  
SUM(case when doquescount>0  and doquescount<50  then 1 else 0 end) as n1,  
SUM(case when doquescount>=50  and doquescount<100  then 1 else 0 end) as n2, 
SUM(case when doquescount>=100  and doquescount<500  then 1 else 0 end) as n3, 
SUM(case when doquescount>=500  and doquescount<1000  then 1 else 0 end) as n4,
SUM(case when doquescount>=1000  and doquescount<1500  then 1 else 0 end) as n5,
SUM(case when doquescount>=1500  and doquescount<2000  then 1 else 0 end) as n6,
SUM(case when doquescount>=2000  and doquescount<2500  then 1 else 0 end) as n7,
SUM(case when doquescount>=2500  and doquescount<3000  then 1 else 0 end) as n8,
SUM(case when doquescount>=3000    then 1 else 0 end) as n9   
from  qz_count 
group by  userid,memberid, domain limit 100;


-- 根据日期分组统计，学员做题量统计
select  daytime,
SUM(case when doquescount=0    then 1 else 0 end) as n0,  
SUM(case when doquescount>=1  and doquescount<50  then 1 else 0 end) as n1,  
SUM(case when doquescount>=50  and doquescount<100  then 1 else 0 end) as n2, 
SUM(case when doquescount>=100  and doquescount<500  then 1 else 0 end) as n3, 
SUM(case when doquescount>=500  and doquescount<1000  then 1 else 0 end) as n4,
SUM(case when doquescount>=1000  and doquescount<1500  then 1 else 0 end) as n5,
SUM(case when doquescount>=1500  and doquescount<2000  then 1 else 0 end) as n6,
SUM(case when doquescount>=2000  and doquescount<2500  then 1 else 0 end) as n7,
SUM(case when doquescount>=2500  and doquescount<3000  then 1 else 0 end) as n8,
SUM(case when doquescount>=3000    then 1 else 0 end) as n9 
from  qz_count 
group by   daytime;


测试
select  count(distinct userid)
from  qz_count  
where  doquescount>=2500  and doquescount<3000;


select  *
from  qz_count  
where  doquescount>=2500  and doquescount<3000;


select  count(distinct userid)
from  qz_count  
where  doquescount>=3000 ;
-- 
-- 根据日期统计每天学员的所做试卷量
select  daytime,
SUM(case when dopapercount=0  then 1 else 0 end) as n0,  
SUM(case when dopapercount>=1  and dopapercount<5  then 1 else 0 end) as n1,  
SUM(case when dopapercount>=5  and dopapercount<10  then 1 else 0 end) as n2, 
SUM(case when dopapercount>=10  and dopapercount<50  then 1 else 0 end) as n3, 
SUM(case when dopapercount>=50  and dopapercount<100  then 1 else 0 end) as n4,
SUM(case when dopapercount>=100  and dopapercount<150  then 1 else 0 end) as n5,
SUM(case when dopapercount>=150  and dopapercount<200  then 1 else 0 end) as n6,
SUM(case when dopapercount>=200  and dopapercount<250  then 1 else 0 end) as n7,
SUM(case when dopapercount>=250  and dopapercount<300  then 1 else 0 end) as n8,
SUM(case when dopapercount>=300    then 1 else 0 end) as n9 
from  paper_count 
group by   daytime  limit 100;


测试检验：
select *  from  paper_count  order by dopapercount desc limit 100;
select  count(*) from paper_count  where   dopapercount>=10  and dopapercount<50   and daytime='2015-04-26'   limit 100;
select  count(distinct userid) from paper_count  where   dopapercount>=10  and dopapercount<50   and daytime='2015-04-26'   limit 100;




 


-- 根据日期统计每天学员在不同区间内答疑数量
每天答疑提问主题 >=60  道以上学员的个数
每天答疑提问主题 >=55 and < 60道以上学员的个数
每天答疑提问主题 >=50 and < 55道以上学员的个数
每天答疑提问主题 >=45  and <50 道以上的学员个数
每天答疑提问主题 >=40  and <45 道以上的学员个数
每天答疑提问主题 >=35 and <40 道以上的学员个数
每天答疑提问主题 >=30 and <35 道以上的学员个数
每天答疑提问主题 >=25 and <30 道以上的学员个数
每天答疑提问主题 >=20 and <25 道以上的学员个数
每天答疑提问主题 >=15 and <20 道以上的学员个数
每天答疑提问主题 >=10 and <15 道以上的学员个数
每天答疑提问主题 >=5   and <10  道以上的学员个数
每天答疑提问主题 >=1   and <5  道以上的学员个数
每天答疑提问主题 =0      道以上的学员个数


select  daytime,
SUM(case when num=0  then 1 else 0 end) as n0,  
SUM(case when num>=1  and num<5  then 1 else 0 end) as n1,  
SUM(case when num>=5  and num<10  then 1 else 0 end) as n2, 
SUM(case when num>=10  and num<15  then 1 else 0 end) as n3, 
SUM(case when num>=15  and num<20  then 1 else 0 end) as n4,
SUM(case when num>=20  and num<25  then 1 else 0 end) as n5,
SUM(case when num>=25  and num<30  then 1 else 0 end) as n6,
SUM(case when num>=30  and num<35  then 1 else 0 end) as n7,
SUM(case when num>=35  and num<40  then 1 else 0 end) as n8,
SUM(case when num>=40  and num<45  then 1 else 0 end) as n9,
SUM(case when num>=45  and num<50  then 1 else 0 end) as n10,
SUM(case when num>=50  and num<55  then 1 else 0 end) as n11,
SUM(case when num>=55  and num<60  then 1 else 0 end) as n12,
SUM(case when num>=60    then 1 else 0 end) as n13 
from  faq_count 
group by   daytime  limit 100;