日志分析hive

hadoop fs -put logs_20160522 /nginx/20160523
date=`/bin/date +%Y%m%d`

/letv/data/apache-hive-0.13.1-bin/bin/hive -e "CREATE EXTERNAL TABLE if not exists nginx_20160620(
 host STRING,  
  uri STRING,  
  request_method STRING,  
  remote_addr STRING,  
  http_marketChannelName STRING,  
  http_hwMac STRING, 
  http_ethMac STRING,   
  http_hwModel STRING,  
  http_appVerName STRING,
  time_local_status STRING,
  body_bytes_sent string,
  bytes_sent string,
  gzip_ratio string,
  http_referer STRING, 
  http_cookie STRING,
  http_user_agent STRING, 
  http_x_forwarded_for STRING,
  upstream_addr STRING,
  upstream_response_time string,
  upstream_status STRING,
 request_time string
)ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n'  location '/nginx/logs’;”

#/letv/data/apache-hive-0.13.1-bin/bin/hive -e "create table nginx_${date}_ChannelUserSort(http_marketChannelName string,UserNum bigint,num bigint);"
#/letv/data/apache-hive-0.13.1-bin/bin/hive -e "insert into table nginx_${date}_ChannelUserSort select *,count(distinct http_hwMac) as UserNum from nginx_${date} group by http_marketChannelName having  UserNum >=50 order by UserNum desc limit 20;"
#/letv/data/apache-hive-0.13.1-bin/bin/hive -e "create table nginx_20160523_UserCount(http_marketChannelName string,UserNum bigint,num bigint);"
#/letv/data/apache-hive-0.13.1-bin/bin/hive -e "insert into table nginx_20160523_UserCount select *,count(distinct http_hwMac) from nginx_20160523;"
/letv/data/apache-hive-0.13.1-bin/bin/hive -e "create table nginx_20160523_ChannelModelTypeSort(http_marketChannelName string,http_hwModel string,ModelTypeNum bigint);"
/letv/data/apache-hive-0.13.1-bin/bin/hive -e "insert into table nginx_20160523_ChannelModelTypeSort select *,count(distinct http_hwModel) as ModelTypeNum from nginx_20160523 group by http_marketChannelName having  ModelTypeNum >=50 order by ModelTypeNum desc limit 20;"

http://blog.csdn.net/lsshlsw/article/details/49155087
http://blog.javachen.com/2015/06/07/spark-configuration.html
http://www.cnblogs.com/phoenixfling/archive/2012/05/09/2492006.html

http://my.oschina.net/tearsky/blog/629201