hive数仓建立时间维表

1.首先利用python中的datetime模块获取到我们想要的日期

import datetime

# 现在的时间
now=datetime.datetime.strptime('2019-01-01','%Y-%m-%d')

# 递增的时间
delta = datetime.timedelta(days=1)
# 1000天后的时间
endnow = now + datetime.timedelta(days=1100)
# 1000天后的时间转换成字符串
endnow = str(endnow.strftime('%Y-%m-%d'))

# 当日期增加到1000天后的日期，循环结束
while str(now.strftime('%Y-%m-%d')) != endnow:
    now += delta
    print(str(now.strftime('%Y-%m-%d')))

2.接着将获取到的日期加载到hdfs中

1）建立存放日期的预处理表

-- 放日期的维表
CREATE EXTERNAL TABLE IF NOT EXISTS tmp_super_date_wml_01
(
    dates                                 STRING COMMENT '日期'
)
COMMENT '时间维表预处理'
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
STORED AS textfile
LOCATION '/user/hive/warehouse/zhidao.db/tmp_super_date_wml_01';

2）将数据导入预处理表中

a.可以看到我们的csv数据文件已经准备就绪

b.好啦！我们在hive的基础日期数据也已经准备就绪！

3.通过sql处理获得我们想要的维表

1）新建时间维表

CREATE EXTERNAL TABLE IF NOT EXISTS dim_zhidao_time_info
(  
date_id                 BIGINT COMMENT '日期ID',
date_mid_desc           STRING COMMENT '中日期',
date_long_desc          STRING COMMENT '长日期',
year_id                 BIGINT COMMENT '年ID',                                    
year_desc               STRING COMMENT '年',             
month_id                STRING COMMENT '月ID',
month_long_desc         STRING COMMENT '长月',
weekday_cn              STRING COMMENT '周几(中文)',           
weekday_eg              STRING COMMENT '周几(英文)',   
week_id                 BIGINT COMMENT '周ID',
week_long_desc          STRING COMMENT 'yyyy年第w周',
daynumber_of_week       BIGINT COMMENT '本周的第几天',    
daynumber_of_year       BIGINT COMMENT '今年的第几天'    
)
COMMENT '时间维表'
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
STORED AS orc
LOCATION '/user/hive/warehouse/zhidao.db/dim_zhidao_time_info';

2）sql处理并写入时间维表

INSERT OVERWRITE TABLE zhidao.dim_zhidao_time_info
SELECT
regexp_replace(dates,'-','') as date_id,
date(dates) as date_mid_desc,
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'yyyy年MM月dd日') as date_long_desc,
year(dates) as year_id,                                                                      -- 年
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'Y年') as year_desc,                  -- 年
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'yyyy-MM') as month_id,
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'yyyy年MM月') month_long_desc,
case when pmod(datediff(dates, '2012-01-01'), 7) = 0 then '星期日'
    when pmod(datediff(dates, '2012-01-01'), 7) = 1 then '星期一'
    when pmod(datediff(dates, '2012-01-01'), 7) = 2 then '星期二'
    when pmod(datediff(dates, '2012-01-01'), 7) = 3 then '星期三'
    when pmod(datediff(dates, '2012-01-01'), 7) = 4 then '星期四' 
    when pmod(datediff(dates, '2012-01-01'), 7) = 5 then '星期五'
    when pmod(datediff(dates, '2012-01-01'), 7) = 6 then '星期六'
    else pmod(datediff(dates, '2012-01-01'), 7) end as weekday_cn,                 -- 周几(中文) 
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'EEEE') as weekday_eg,    -- 周几(英文)    
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'w') week_id,
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'yyyy年第w周') week_long_desc,
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'u') as daynumber_of_week,     -- 本周的第几天
from_unixtime(unix_timestamp(dates,'yyyy-MM-dd'),'D') as daynumber_of_year     -- 今年的第几天

from zhidao.tmp_super_date_wml_01;

3）检验数据结果