2、实现基于纯Hive SQL的ETL过程,从TRLOG表生成ALLOG表;(结果是一套SQL)
答案:
1.
UDF
package org.apache.hadoop.hive.udf; public class RowNumber extends org.apache.hadoop.hive.ql.exec.UDF { private static int MAX_VALUE = 50; private static String comparedColumn[] = new String[MAX_VALUE]; private static int rowNum = 1; public int evaluate(Object... args) { String columnValue[] = new String[args.length]; for (int i = 0; i < args.length; i++) columnValue[i] = args[i].toString(); if (rowNum == 1) { for (int i = 0; i < columnValue.length; i++) comparedColumn[i] = columnValue[i]; } for (int i = 0; i < columnValue.length; i++) { if (!comparedColumn[i].equals(columnValue[i])) { for (int j = 0; j < columnValue.length; j++) { comparedColumn[j] = columnValue[j]; } rowNum = 1; return rowNum++; } } return rowNum++; } public static void main(String[] args) { RowNumber aRowNumber = new RowNumber(); System.out.println(aRowNumber.evaluate("12332321")); System.out.println(aRowNumber.evaluate("12332321")); System.out.println(aRowNumber.evaluate("12332321")); System.out.println(aRowNumber.evaluate("12332321")); System.out.println(aRowNumber.evaluate("12332321")); } }
INSERT OVERWRITE TABLE ALLOG
SELECT t1.platform,t1.user_id,RowNumber(t1.user_id)seq,t2.click_url FROM_URL,t1.click_url TO_URL FROM
(select *,RowNumber(user_id)seq from trlog)t1
LEFT OUTER JOIN
(select *,RowNumber(user_id)seq from trlog)t2
on t1.user_id = t2.user_id and t1.seq=t2.seq+1;
2.
INSERT OVERWRITE TABLE ALLOG
SELECT t1.platform,t1.user_id,t1.seq,t2.click_url FROM_URL,t1.click_url TO_URL FROM
(SELECT platform,user_id,click_time,click_url,count(1) seq FROM (SELECT a.*,b.click_time click_time1,b.click_url click_url2 FROM trlog a left outer join trlog b on a.user_id = b.user_id)t WHERE click_time>=click_time1 GROUP BY platform,user_id,click_time,click_url)t1
LEFT OUTER JOIN
(SELECT platform,user_id,click_time,click_url,count(1) seq FROM (SELECT a.*,b.click_time click_time1,b.click_url click_url2 FROM trlog a left outer join trlog b on a.user_id = b.user_id)t WHERE click_time>=click_time1 GROUP BY platform,user_id,click_time,click_url )t2
on t1.user_id = t2.user_id and t1.seq = t2.seq + 1;