第三阶段模块二作业
作业题：
1.用拉链表实现核心交易分析中DIM层商家维表，并实现该拉链表的回滚（自己构造数据，编写SQL，并要有相应的文字说明）
2.在会员分析中计算沉默会员数和流失会员数
沉默会员的定义：只在安装当天启动过App，而且安装时间是在7天前
流失会员的定义：最近30天未登录的会员
3. 在核心交易分析中完成如下指标的计算
统计2020年每个季度的销售订单笔数、订单总额
统计2020年每个月的销售订单笔数、订单总额
统计2020年每周（周一到周日）的销售订单笔数、订单总额
统计2020年国家法定节假日、休息日、工作日的订单笔数、订单总额
拉链表维护商家信息
1.创建拉链维表
-- 创建商家信息维表
drop table if exists dim.dim_trade_shops;
create table dim.dim_trade_shops(
`shopid` int COMMENT '商铺ID',
`userid` int COMMENT '商铺负责人',
`areaid` int COMMENT '区域ID',
`shopname` string COMMENT '商铺名称',
`shoplevel` int COMMENT '商铺等级',
`status` int COMMENT '商铺状态',
`createtime` string COMMENT '创建日期',
`modifytime` string COMMENT '修改日期',
`startdate` string COMMENT '生效起始日期',
`enddate` string COMMENT '失效结束日期'
) COMMENT '商家信息表';
--创建商家信息表(增量表 分区表)
drop table if exists ods.ods_trade_shops;
create table ods.ods_trade_shops(
`shopid` int COMMENT '商铺ID',
`userid` int COMMENT '商铺负责人',
`areaid` int COMMENT '区域ID',
`shopname` string COMMENT '商铺名称',
`shoplevel` int COMMENT '商铺等级',
`status` int COMMENT '商铺状态',
`createtime` string COMMENT '创建日期',
`modifytime` string COMMENT '修改日期'
) COMMENT '商家信息表'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by ',';
2.准备数据
/root/data/shops/shop-2020-11-20.dat
100050,1,100225,WSxxx营超市,1,1,2020-06-28,2020-11-20 13:22:22
100052,2,100236,新鲜xxx旗舰店,1,1,2020-06-28,2020-11-20 13:22:22
100053,3,100011,华为xxx旗舰店,1,1,2020-06-28,2020-11-20 13:22:22
100054,4,100159,小米xxx旗舰店,1,1,2020-06-28,2020-11-20 13:22:22
100055,5,100211,苹果xxx旗舰店,1,1,2020-06-28,2020-11-20 13:22:22
/root/data/shops/shop-2020-11-21.dat
100057,7,100311,三只xxx鼠零食,1,1,2020-06-28,2020-11-21 13:22:22
100058,8,100329,良子xxx铺美食,1,1,2020-06-28,2020-11-21 13:22:22
100054,4,100159,小米xxx旗舰店,2,1,2020-06-28,2020-11-21 13:22:22
100055,5,100211,苹果xxx旗舰店,2,1,2020-06-28,2020-11-21 13:22:22
/root/data/shops/shop-2020-11-22.dat
100059,9,100225,乐居xxx日用品,1,1,2020-06-28,2020-11-22 13:22:22
100060,10,100211,同仁xxx大健康,1,1,2020-06-28,2020-11-22 13:22:22
100052,2,100236,新鲜xxx旗舰店,1,2,2020-06-28,2020-11-22 13:22:22
3.加载数据到增量表（2020-11-20）
load data local inpath '/root/data/shops/shop-2020-11-20.dat' into table ods.ods_trade_shops partition(dt='2020-11-20');
4.初始化拉链表数据（2020-11-20）
insert overwrite table dim.dim_trade_shops
select shopid, userid, areaid, shopname, shoplevel,
status, createtime, modifytime,
case when modifytime is not null then substr(modifytime,0,10)
else substr(createtime,0,10) end as startdate,
'3000-12-31' as enddate
from ods.ods_trade_shops
where dt ='2020-11-20';
5.导入2020-11-21 的数据，更新拉链表数据
load data local inpath '/root/data/shops/shop-2020-11-21.dat' into table ods.ods_trade_shops partition(dt = '2020-11-21');
insert overwrite table dim.dim_trade_shops
select shopid, userid, areaid, shopname,
shoplevel, status, createtime, modifytime,
CASE WHEN modifytime is not null THEN substr(modifytime,0,10)
ELSE substr(createtime,0,10) END AS startdate,
'3000-12-31' AS enddate
from ods.ods_trade_shops where dt = '2020-11-21'
union all
select b.shopid, b.userid ,b.areaid , b.shopname, b.shoplevel,
b.status, b.createtime, b.modifytime, b.startdate,
CASE WHEN a.shopid is not null and b.enddate ='3000-12-31'
THEN date_add('2020-11-21',-1)
ELSE b.enddate end as enddate
from (select * from ods.ods_trade_shops where dt='2020-11-21') a
right join dim.dim_trade_shops b on a.shopid = b.shopid;
6.导入2020-11-22 的数据，更新拉链表数据
load data local inpath '/root/data/shops/shop-2020-11-22.dat' into table ods.ods_trade_shops partition(dt = '2020-11-22');
insert overwrite table dim.dim_trade_shops
select shopid, userid, areaid, shopname,
shoplevel, status, createtime, modifytime,
CASE WHEN modifytime is not null THEN substr(modifytime,0,10)
ELSE substr(createtime,0,10) END AS startdate,
'3000-12-31' AS enddate
from ods.ods_trade_shops where dt = '2020-11-22'
union all
select b.shopid, b.userid ,b.areaid , b.shopname, b.shoplevel,
b.status, b.createtime, b.modifytime, b.startdate,
CASE WHEN a.shopid is not null and b.enddate ='3000-12-31'
THEN date_add('2020-11-22',-1)
ELSE b.enddate end as enddate
from (select * from ods.ods_trade_shops where dt='2020-11-22') a
right join dim.dim_trade_shops b on a.shopid = b.shopid;
7.创建加载商家信息表的数据脚本
/root/script/shops/dim_load_shops.sh
#!/bin/bash
source /etc/profile
if [ -n "$1" ]
then
do_date=$1
else
do_date=`date -d "-1 day" +%F`
fi
sql="
insert overwrite table dim.dim_trade_shops
select shopid, userid, areaid, shopname,
shoplevel, status, createtime, modifytime,
CASE WHEN modifytime is not null THEN substr(modifytime,0,10)
ELSE substr(createtime,0,10) END AS startdate,
'3000-12-31' AS enddate
from ods.ods_trade_shops where dt = '$do_date'
union all
select b.shopid, b.userid ,b.areaid , b.shopname, b.shoplevel,
b.status, b.createtime, b.modifytime, b.startdate,
CASE WHEN a.shopid is not null and b.enddate ='3000-12-31'
THEN date_add('$do_date',-1)
ELSE b.enddate end as enddate
from (select * from ods.ods_trade_shops where dt='$do_date') a
right join dim.dim_trade_shops b on a.shopid = b.shopid;
"
hive -e "$sql"
8.拉链表回滚实现
以回滚到2020-11-21日的数据为例
drop table if exists tmp.shops_tmp;
create table if not exists tmp.tmp_shops as
select shopid, userid, areaid, shopname,
shoplevel, status, createtime,
modifytime, startdate, enddate
from dim.dim_trade_shops
where enddate < '2020-11-21'
union all
select shopid, userid, areaid, shopname,
shoplevel, status, createtime,
modifytime, startdate, '3000-12-31' as enddate
from dim.dim_trade_shops
where startdate <= '2020-11-21' and enddate >= '2020-11-21';
insert overwrite table dim.dim_trade_shops select * from tmp.tmp_shops;
创建回滚脚本
#!/bin/bash
source /etc/profile
if [ -n "$1" ]
then
do_date=$1
else
do_date=`date -d "-1 day" +%F`
fi
sql="
drop table if exists tmp.shops_tmp;
create table if not exists tmp.tmp_shops as
select shopid, userid, areaid, shopname,
shoplevel, status, createtime,
modifytime, startdate, enddate
from dim.dim_trade_shops
where enddate < '$do_date'
union all
select shopid, userid, areaid, shopname,
shoplevel, status, createtime,
modifytime, startdate, '3000-12-31' as enddate
from dim.dim_trade_shops
where startdate <= '$do_date' and enddate >= '$do_date';
insert overwrite table dim.dim_trade_shops select * from tmp.tmp_shops;
"
hive -e "$sql"
计算沉默会员数和流失会员数
1.计算沉默会员
-- 先获取只启动一次的会员 在过滤出当前会员的登陆时间小于七天前的日期
select count(*) clientNum from dws.dws_member_start_day where dt <= date_add(current_date,-7) and device_id in(select device_id from (select device_id, count(*) cnt from dws.dws_member_start_day group by device_id having cnt = 1) tmp);
2.计算流失会员
select count(distinct device_id) lossNum from dws.dws_member_start_day where device_id not in (select distinct device_id from dws.dws_member_start_day where dt >=date_add(current_date,30))
hive.strict.checks.cartesian.product=false;
销售指标统计
先统计每天的销售订单笔数和每日订单总额
订单状态 -3 用户拒收 -2未付款的订单 -1用户取消 0 待发货 1配送中 2用户确认收货
订单有效标志 -1 删除 1 有效
drop table if exists dws.dws_trade_orders_day;
create table if not exists dws.dws_trade_orders_day(
day_dt string comment '日期：yyyy-MM-dd',
day_cnt decimal commnet '日订单笔数',
day_sum decimal comment '日订单总额'
) comment '日订单统计表';
select dt, count(*) cnt,sum(totalMoney) sm from (select distinct orderid,dt,totalMoney from dwd.dwd_trade_orders where status >= 0 and dataFlag = '1') tmp group by dt;
insert overwrite table dws.dws_trade_orders_day select dt, count(*) cnt, sum(totalMoney) sm from (select distinct orderid, dt, totalMoney from dwd.dwd_trade_orders where status >= 0 and dataFlag = '1') tmp group by dt;
select * from dws.dws_trade_orders_day where day_dt between '2020-01-01' and '2020-12-31';
1.统计2020年每个季度的销售订单笔数、订单总额
drop table if exists dws.dws_trade_orders_quarter;
create table if not exists dws.dws_trade_orders_quarter(
year string comment '年份',
quarter string comment '季度',
cnt decimal comment '订单总笔数',
sum decimal comment '订单总额'
) comment '季度订单统计表';
insert overwrite table dws.dws_trade_orders_quarter
with tmp as(
select substr(day_dt,0,4) year,
case when substr(dat_dt,6,2)="01" or substr(dat_dt,6,2)="02" or
substr(day_dt,6,2)="03" then "1"
when substr(dat_dt,6,2)="04" or substr(dat_dt,6,2)="05" or
substr(day_dt,6,2)="06" then "2"
when substr(dat_dt,6,2)="07" or substr(dat_dt,6,2)="08" or
substr(day_dt,6,2)="09" then "3"
when substr(dat_dt,6,2)="10" or substr(dat_dt,6,2)="11" or
substr(day_dt,6,2)="12" then "4" as quarter
day_cnt,day_sum
from dws.dws_trade_orders_day)
select year,quarter,sum(day_cnt),sum(day_sum) from tmp group by year quarter;
2.统计2020年每个月的销售订单笔数，订单总额
drop table if exists dws.dws_trade_orders_month;
create table if not exists dws.dws_trade_orders_month(
yearstring comment '年份',
month string comment '月份',
month_cnt decimal comment '月订单总笔数',
month_sum decimal comment '月订单总额'
) comment '月订单统计表';
insert overwrite table dws.dws_trade_orders_month
with tmp as(
select substr(day_dt,0,4) year,
sunstr(day_dt,6,2) month,
day_cnt, day_sum
from dws.dws_trade_orders_day)
select year,month,sum(day_cnt) month_cnt,sum(day_sum) month_sum
from tmp group by year,month;
3.统计2020年每周（周一到周日）的销售订单笔数，订单总额
设计自定函数，用于祭祀日期属于第几周
1.创建maven工程，添加依赖
org.apache.hive
hive-exec
2.3.7
2.新建WeekOfYear类，继承UDF ，实现evaluate
public IntWriteble evaluate(final Text datestr){
String today = datestr.toString();
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
Date date = null;
try{
date = formar.parse(today);
} catch(ParseExecption e){
e.printStackTrace();
}
Calendar calendar = Calendar.getInstance();
calendar.setFirstDayOfWeek(Calendar.MONDAY);
calendar.setTime(date);
int i = calendar.get(Calendar.WEEK_OF_YEAR);
return new IntWritable(i);
}
3.将项目打包，并上传至服务器
4.添加jar包到hive, 创建临时函数，即可使用
add jar /root/data/jars/udf-weekofyear-1.0-SANPSHOT.jar;
create temporary function weekofyear as "com.haozhen.udf.WeekOfYear";
5.编写相关sql
drop table if exists dws.dws_trade_orders_week;
create table if not exists dws.dws_trade_orders_week(
year string comment '年份',
week string comment '一年中的第几周',
week_cnt decimal comment '周订单总笔数',
week_sum decimal comment '周订单总额'
) comment '周订单统计表';
insert overwrite table dws.dws_trade_orders_week
select substr(day_dt,0,4) year,
weekofyear(day_dt) week,
sum(day_cnt), sum(day_sum)
from dws.dws_trade_orders_day group by substr(day_dt,0,4) year,
weekofyear(day_dt) week;
统计2020国家法定节假日、休息日、工作日的订单笔数、订单总额
1.创建日期信息维表，记录日期是否是节假日，工作日
drop table if exists dim.dim_day_info;
create table if not exists dim.dim_day_info(
day_dt string comment '日期',
is_holidays int comment '节假日标识： 0不是 1是',
is_workday int comment '工作日标识 0不是 1是'
) comment '日期信息表';
2初始化日期信息表
3.根据日期信息维表和日订单统计表统计
-- 统计2020节假日的订单笔数，订单总额
select nvl(sum(day_cnt),0) nvl(sum(day_sum),0) from
dws.dws_trade_orders_day A left join dim.dim_day_info B on A.day_dt = B.day_dt where B.is_holiday = 1;
-- 统计2020年休息日的订单笔数，订单总额
select nvl(sum(day_cnt),0) nvl(sum(day_sum),0) from
dws.dws_trade_orders_day A left join dim.dim_day_info B on A.day_dt = B.day_dt where B.is_workday = 0;
-- 统计2020节工作日的订单笔数，订单总额
select nvl(sum(day_cnt),0) nvl(sum(day_sum),0) from
dws.dws_trade_orders_day A left join dim.dim_day_info B on A.day_dt = B.day_dt where B.is_workday = 1;