Hive学习五--日志案例分析

Hive学习五–日志案例分析

标签（空格分隔）： Hive

一，统计分析每日各时段的PV和UV
1：创建数据库

drop database if exists db_track;
create database db_track;

2，创建表(建hive表，表列分隔符和文件保持一致)

drop table if exists db_track.track_log ;
create table db_track.track_log(
id                string,
url               string,
referer           string,
keyword           string,
type              string,
guid              string,
pageId            string,
moduleId          string,
linkId            string,
attachedInfo      string,
sessionId         string,
trackerU          string,
trackerType       string,
ip                string,
trackerSrc        string,
cookie            string,
orderCode         string,
trackTime         string,
endUserId         string,
firstLink         string,
sessionViewNo     string,
productId         string,
curMerchantId     string,
provinceId        string,
cityId            string,
fee               string,
edmActivity       string,
edmEmail          string,
edmJobId          string,
ieVersion         string,
platform          string,
internalKeyword   string,
resultSum         string,
currentPage       string,
linkPosition      string,
buttonPosition    string
)
partitioned by(date string, hour string)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ;

3,load数据到hive表中

load data local inpath '/opt/datas/2015082818' overwrite into table db_track.track_log partition(date='20150828',hour='18');

load data local inpath '/opt/datas/2015082819' overwrite into table db_track.track_log partition(date='20150828',hour='19');

4,写Hive sql统计，结果落地到hive表daily_hour_visit,创建表daily_hour_visit表

drop table if exists db_track.daily_hour_visit ;
create table db_track.daily_hour_visit( date string, hour string, uv string, pv string ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ;

5,将统计结果写入到daily_hour_visit表中

insert into table db_track.daily_hour_visit select date,hour,count(url) pv, count(distinct guid) uv from db_track.track_log where date='20150828' group by date,hour;

6， mysql创建数据库和数据表（从hive表2导出结果数据到mysql目的表）

create database db_track;

drop table db_track.track_log if exists db_track.track_log;
create table db_track.track_log( date varchar(255) not null, hour varchar(255) default null, pv varchar(255) default null, uv varchar(255) default null, primary key(date,hour) );

7,sqoop导出数据

bin/sqoop export \
--connect jdbc:mysql://localhost:3306/db_track \ --username root \ --password root \ --table track_log \ --num-mappers 1 \ --export-dir /user/hive/warehouse/db_track.db/daily_hour_visit \ --fields-terminated-by '\t'