hiveSQL记录

CREATE EXTERNAL TABLE ks3_nginx_pv(
host_ip string,
idc string,
upstream_ip string,
bucket_owner string,
`bucket` string,
time bigint,
time_id  int ,
remote_ip string,
requester string,
request_id string,
operation string,
key string,
request_uri string,
http_status  int ,
error_code string,
bytes_sent bigint,
object_size bigint,
total_time bigint,
turn_around_time bigint,
referer string,
user_agent string,
version_id string,
request_body_size  int  COMMENT  '上传的数据大小' ,
response_size  int  COMMENT  '返回数据的大小' ,
province string,
logtype string COMMENT  'app or CDN NAME' ,
cisp string COMMENT  'client isp' ,
sisp string COMMENT  'server isp' ,
extend string
 )
PARTITIONED BY(day  int )
ROW FORMAT SERDE  "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
STORED AS RCFILE

LOCATION '/user/hive/warehouse/tables/ks3_nginx_pv'







adccompany string
av string
imei string
imsi string
model string
phone string
pkgname string
type int
upload_time string






create EXTERNAL table IF NOT EXISTS userInfo (id int,sex string, age int, name string, email string,sd string, ed string)  
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' location '/hive/dw';


create EXTERNAL table phone_sign (
  upload_time string,
  model string ,
  adccompany string ,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  phone string ,
  type int
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/phonesign';

alter table phone_sign add partition (dt='201401') location '/user/hdfs/source/log/phonesign/201401';

show partitions phone_sign;

create EXTERNAL table dot_url (
  cur_time string,
  phone string ,
  model string ,
  adccompany string ,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ossdk  string,
  url  string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/doturl';

create EXTERNAL table stay_time (
  cur_time string,
  model string ,
  adccompany string ,
  ossdk  string,
  av string ,
  imsi string ,
  ip  string,
  stay_time string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/staytime';

create EXTERNAL table success_show (
  cur_time string,
  model string ,
  adccompany string ,
  ossdk  string,
  av string ,
  imsi string ,
  ip  string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/successshow';


create EXTERNAL table dot_action (
  imsi string,
  adccompany string ,
  pkgname string ,
  ip  string,
  model string ,
  phone string ,
  cur_time  string,
  calltype  int,
  actiontype  int,
  result  int,
  ossdk  string,
  av  string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/dotaction';


java -jar sign-phone-stat-0.0.1-SNAPSHOT.jar com.winksi.mis.service.Main




dot_click.log
create EXTERNAL table dot_click (
  cur_time string,
  phone string ,
  model string ,
  adccompany string ,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ossdk  string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/dotclick';

create EXTERNAL table phone_click (
  cur_time string,
  model string ,
  adccompany string ,
  ossdk  string,
  av string ,
  imsi string ,
  ip  string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/phoneclick';



select model,adccompany,ossdk,av,substr(cur_time, 0, 8),phone,count(distinct(imsi)) from dot_action  
where dt = '201401' and substr(cur_time, 0, 8) = '20140115' 
group by model,adccompany,ossdk,av,phone,substr(cur_time, 0, 8),imsi


 qymp_20140121.dat.c09_8101文件的字段（ossdk还没有,也没有AV）
imsi,adccompany, phone,  cur_time,  call_length,  call_type,  show_length ,pkgname,ip,model,  ossdk,av




create EXTERNAL table qymp (
  imsi string,
  adccompany string ,
  phone string ,
  cur_time  string,
  call_length string ,
  call_type string , //通话类型【（1呼出未接通；2呼出接通；3呼入未接通；4呼入接通）】
  show_length  string,
  pkgname string,
  ip string,
  model string,
  av string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/qymp';
460017720510813,100212,114,20130901133521,147,2,0,com.android.phone,121.31.247.219
999030044812828,100422,10010,20140531134849,16,2,0,com.android.phone,61.139.151.142,C10,4.3.3.0.20131024
460028601338694,100422,10086,20140531103253,47,2,0,com.android.phone,221.7.7.236,intki-E87,4.3.3.0.20130620


hive -S -e 'select model,adccompany,count(distinct(imsi)) from qymp where model is not null group by model,adccompany' > result.txt








//日期，企业ID，一级分类ID，二级分类ID，省ID，市ID，机型，渠道，ossdk，av,imsi,imei,ip,p
sb.append(sdf.format(date)).append(",").append(dotReqBean.getEnterpriseId()).append(",")
.append(parentId).append(",").append(dotReqBean.getCategory())
.append(",").append(area.getPareaid()).append(",").append(dotReqBean.getCityId())
.append(StringEncode.encodeStr(dotReqBean.getModel()))
.append(",").append(dotReqBean.getAdccompany()).append(",").append(dotReqBean.getOssdk())
.append(",").append(dotReqBean.getAv()).append(",").append(dotReqBean.getImsi()).append(",")
.append(dotReqBean.getImei()).append(",").append(ip).append(",").append(dotReqBean.getP());


-------------------------v2
create EXTERNAL table click_addr (
  cur_time string,
  enterpriseId int,
  onetype int,
  twotype int,
  provinceId int,
  cityId int,
  model string ,
  adccompany string ,
  ossdk string,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ip string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/clickaddr';


alter table click_addr add partition (dt='201401') location '/user/hdfs/source/log/clickaddr/201401';

create EXTERNAL table click_call (
  cur_time string,
  enterpriseId int,
  onetype int,
  twotype int,
  provinceId int,
  cityId int,
  model string ,
  adccompany string ,
  ossdk string,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ip string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/clickcall';

create EXTERNAL table click_website (
  cur_time string,
  enterpriseId int,
  onetype int,
  twotype int,
  provinceId int,
  cityId int,
  model string ,
  adccompany string ,
  ossdk string,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ip string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/clickwebsite';

create EXTERNAL table click_weibo (
  cur_time string,
  enterpriseId int,
  onetype int,
  twotype int,
  provinceId int,
  cityId int,
  model string ,
  adccompany string ,
  ossdk string,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ip string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/clickweibo';


create EXTERNAL table enter_page (
  cur_time string,
  enterpriseId int,
  onetype int,
  twotype int,
  provinceId int,
  cityId int,
  model string ,
  adccompany string ,
  ossdk string,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ip string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' 
    LOCATION '/user/hdfs/source/log/enterpage';
alter table enter_page add partition (dt='201401') location '/user/hdfs/source/log/enterpage/201401';

create EXTERNAL table more_enterprise (
  cur_time string,
  enterpriseId int,
  onetype int,
  twotype int,
  provinceId int,
  cityId int,
  model string ,
  adccompany string ,
  ossdk string,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ip string,
  phone string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/moreenterprise';



create EXTERNAL table rec_detail (
  cur_time string,
  enterpriseId int,
  onetype int,
  twotype int,
  provinceId int,
  cityId int,
  model string ,
  adccompany string ,
  ossdk string,
  av string ,
  pkgname string ,
  imsi string ,
  imei string ,
  ip string,
  phone string,
  position int  
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/log/recdetail';

//终端活跃数表
create EXTERNAL table term_active (
  imsi string ,
  adccompany string ,
  cur_time string,
  showcc string,
  ip string,
  av string,
  model string  
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY ';'
  LOCATION '/user/hdfs/source/log/companytermactive';
  
  alter table term_active add partition (dt='201402') location '/user/hdfs/source/log/companytermactive/201402'
alter table term_active add partition (dt='201403') location '/user/hdfs/source/log/companytermactive/201403'
alter table term_active add partition (dt='201404') location '/user/hdfs/source/log/companytermactive/201404'
alter table term_active add partition (dt='201405') location '/user/hdfs/source/log/companytermactive/201405'
alter table term_active add partition (dt='201406') location '/user/hdfs/source/log/companytermactive/201406'
alter table term_active add partition (dt='201407') location '/user/hdfs/source/log/companytermactive/201407'
alter table term_active add partition (dt='201408') location '/user/hdfs/source/log/companytermactive/201408'
alter table term_active add partition (dt='201409') location '/user/hdfs/source/log/companytermactive/201409'
alter table term_active add partition (dt='201410') location '/user/hdfs/source/log/companytermactive/201410'
alter table term_active add partition (dt='201411') location '/user/hdfs/source/log/companytermactive/201411'
alter table term_active add partition (dt='201412') location '/user/hdfs/source/log/companytermactive/201412'
alter table term_active add partition (dt='201301') location '/user/hdfs/source/log/companytermactive/201301'
alter table term_active add partition (dt='201302') location '/user/hdfs/source/log/companytermactive/201302'
alter table term_active add partition (dt='201303') location '/user/hdfs/source/log/companytermactive/201303'
alter table term_active add partition (dt='201304') location '/user/hdfs/source/log/companytermactive/201304'
alter table term_active add partition (dt='201305') location '/user/hdfs/source/log/companytermactive/201305'
alter table term_active add partition (dt='201306') location '/user/hdfs/source/log/companytermactive/201306'
alter table term_active add partition (dt='201307') location '/user/hdfs/source/log/companytermactive/201307'
alter table term_active add partition (dt='201308') location '/user/hdfs/source/log/companytermactive/201308'
alter table term_active add partition (dt='201309') location '/user/hdfs/source/log/companytermactive/201309'
alter table term_active add partition (dt='201310') location '/user/hdfs/source/log/companytermactive/201310'
alter table term_active add partition (dt='201311') location '/user/hdfs/source/log/companytermactive/201311'
alter table term_active add partition (dt='201312') location '/user/hdfs/source/log/companytermactive/201312'






select cust.CUST_ID ,cust.BIZ_PHONES from CUST_BIZ biz,CUST cust where biz.BIZ_TYPE_ID = 1 and biz.CUST_ID = cust.CUST_ID;


do
        echo "select cust.CUST_ID ,cust.BIZ_PHONES from CUST_BIZ biz,CUST cust where biz.BIZ_TYPE_ID = 1 and biz.CUST_ID = cust.CUST_ID;" | mysql -N --default-character-set=utf8  -h172.16.1.81 -P3308 
-uadmin -ptonggangdasha custmis  > cust_biz/cust_phone.txt 


done < cust.table


//02133189900 460006052730784 100151 2014-05-21 57 2 0 com.byread.reader 117.136.8.80 HTCDesireS 20130718
create EXTERNAL table call_show_zdws (
  phone string,
  imsi string ,
  adccompany string ,
  cur_time string,
  call_length string ,
  call_type string , //通话类型【（1呼出未接通；2呼出接通；3呼入未接通；4呼入接通）】
  show_length  string,
  pkgname string,
  ip string,
  model string,
   av string,
  enterpriseId string  
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
  LOCATION '/user/hdfs/combine/ZdwsCombine/';


  
  //010,20140609,1032640,110000,25,1,4,460010902589465,100212,Coolpad7295C,4.3.3.0.20130608
  //phone,curTime,enterId,cityId,categroyId,provinceId,oneType,imsi,adc,model,av;
  create EXTERNAL table call_show_zdws (
  phone string,
  cur_time string,
  enterpriseId string,
  cityId string,
  categroyId string,
  provinceId string,
  oneType string,
  imsi string ,
  adccompany string ,
  model string,
  av string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
  LOCATION '/user/hdfs/result/dotmatch/';
  
  
  
  create EXTERNAL table userzone (
  imsi string,
  zoneCode string
  ) 
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/combine/UserZone/';
删除分区
ALTER TABLE call_show_zdws  DROP PARTITION (dt='20140624')

//2014-07-09 11:09:41,00,460,30993,4119,131,39.9194,116.378547,110000
  create EXTERNAL table location_info (
  cur_time string,
  mobileNetworkCode string,
  mobileCountryCode string,
  cellId string,
  locationAreaCode string,
  baiduAreaId string,
  lat string,
  lng string,
  areaId string
  )  partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/source/222log/location/';

999038509201122,4.3.3.0.20131024,100422,ADVANS3A,
  create EXTERNAL table registrations (
  imsi string,
  av string,
  adc string,
  model string
  )  partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
    LOCATION '/user/hdfs/result/RegistrationsJob/';



//终端活跃数表
create EXTERNAL table active (
  tid string ,
  city_id  string ,
  imsi string ,
  adc string,
  model string,
  av string,
  report_date string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' 
  LOCATION '/user/hdfs/combine/ActiveJob/';


//终端活跃数表
create EXTERNAL table active_month (
  tid string ,
  city_id  string ,
  adc string,
  model string,
  av string,
  report_month string,
  num string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' 
  LOCATION '/user/hdfs/result/ActiveMonth/';
  
  http://172.16.1.50:7180 admin wxwk)@!#
  


//安全结算表
create EXTERNAL table safe_stat (
  report_date string,
  ip string,
  tid string ,
  imsi string,
  imei string ,
  adc string,
  model string,
  av string,
  pkgname string,
  an string,
  city_id  string ,
  action_time string,
  type string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' 
  LOCATION '/user/hdfs/source/log/safecenter_menu/';
  
  //安全结算表
  2014-08-14 14:47:03,218.240.33.174,s460028014502919,460028014502919,356489055686682,100242,
  AOSPonMako,APKPRJ94_2.2.2.43_20140814,com.ishow.client.android.plugin.company,
  w-Android-gen-768x1184,110000,20140814144507,5
create EXTERNAL table bdsafe_install (
  report_date string,
  ip string,
  tid string ,
  imsi string,
  imei string ,
  adc string,
  model string,
  av string,
  pkgname string,
  an string,
  city_id  string ,
  action_time string,
  type string
  ) partitioned by (dt string)
  ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' 
  LOCATION '/user/hdfs/source/222log/bdSafeInstall/';