Doris bitmap实现留存计算
一、背景
什么是留存 比如用户今天使用某应用，明天还继续使用该应用，叫做留存。
bitmap原理 基本思想是用数组下标表示元素的值，用0,1标记元素是否存在，1表示存在，0表示不存在。由于使用bit为单位来存储数据，大大节省了存储空间。例如：给定一个数组array=[1,3,5];
为什么要用bitmap？bitmap为什么能计算留存？bitmap为什么只支持int类型？
①节省存储空间。
②留存实际上是求交集，bit可以进行与运算。bitmap不仅仅可以计算留存，还可以计算漏斗、新增、活跃，灵活使用bitmap的交并差集。
③数组下标代表的是元素的值。
二、函数
bitmap_hash()：对任意非整型的输入计算32位的哈希值，返回包含该hash值的bitmap。
to_bitmap()：bigint转换为bitmap，输出包含该元素的bitmap。
bitmap_intersect(bitmap)：输入一组bitmap，求交集并返回一个新的bitmap。
bitmap_union(bitmap)：输入一组bitmap，求并集，并返回新的bitmap。
bitmap_count()：返回count值。
intersect_count(expr1,expr2,expr3)：求交集。expr1是 求交集Bitmap 列，expr2是用来过滤的维度列， expr3开始是变长参数，含义是过滤维度列的不同取值。
bitmap_and(bitmap expr1,bitmap expr2)：计算两个bitmap的交集，返回新的bitmap。
hll_hash()：任意类型转换hll类型。
hll_union_agg(hll)：计算count distinct。
三、count distinct 与bitmap精确去重
-- 1.传统count distinct
-- 40475
select
count(distinct user_id),
bitmap_count(bitmap_union(to_bitmap(user_id))),
BITMAP_UNION_COUNT(to_bitmap(user_id))
from gx_doris_game_login_day_agg
where actday ='2020-10-02'
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'
-- 2.bitmap精确去重
-- 40475
select
bitmap_count(user_id)
from gx_doris_game_login_day_agg_bitmap
where actday in ('2020-10-02')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'四、留存计算方法
count distinct 传统的count distinct精确去重可以保留明细数据，灵活性较高，但是Doris是基于MPP架构，在计算过程中需要多次进行数据shuffle，性能会随着数据量增大而下降。
SELECT
a.actday,
count(distinct a.user_id) as active,
count(distinct case when datediff(b.actday,a.actday)=1 then b.user_id end) as retention,
count(distinct case when datediff(b.actday,a.actday)=2 then b.user_id end) as retention2
from
(
select
actday,user_id,appkey
from gx_doris_game_login_day_agg
where actday ='2020-10-02'
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'
)a
left join
(
select
actday,user_id,appkey
from gx_doris_game_login_day_agg
where actday in ('2020-10-03','2020-10-04')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'
)b on a.user_id=b.user_id
group by a.actdaybitmap 假如给定一个数组A, 对该数组去重, 初始化bitmap为全0; 逐个处理数组A的元素, 以A中元素取值作为bitmap的下标, 将该下标的bit置1; 最后统计bitmap中1的个数即为数组A的count distinct结果。
-- int类型
select
'2020-10-02' actday,
intersect_count(to_bitmap(user_id),actday,'2020-10-02') as active,
-- intersect_count(to_bitmap(user_id),actday,'2020-10-03') as user03,
intersect_count(to_bitmap(user_id),actday,'2020-10-02','2020-10-03') as retention1,
intersect_count(to_bitmap(user_id),actday,'2020-10-02','2020-10-04') as retention2
from gx_doris_game_login_day_agg_1
where actday in ('2020-10-02','2020-10-03','2020-10-04')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'-- string类型：user_id
-- 40475 32476 31688
-- 0.375s
select
'2020-10-02' actday,
intersect_count(bitmap_hash(user_id),actday,'2020-10-02') as active,
-- intersect_count(to_bitmap(user_id),actday,'2020-10-03') as user03,
intersect_count(bitmap_hash(user_id),actday,'2020-10-02','2020-10-03') as retention1,
intersect_count(bitmap_hash(user_id),actday,'2020-10-02','2020-10-04') as retention2
from gx_doris_game_login_day_agg
where actday in ('2020-10-02','2020-10-03','2020-10-04')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'-- string类型：m1
-- 2020-10-02 16907 12502 11611
-- 0.288s
select
'2020-10-02' actday,
intersect_count(bitmap_hash(m1),actday,'2020-10-02') as active,
-- intersect_count(to_bitmap(m1),actday,'2020-10-03') as user03,
intersect_count(bitmap_hash(m1),actday,'2020-10-02','2020-10-03') as retention1,
intersect_count(bitmap_hash(m1),actday,'2020-10-02','2020-10-04') as retention2
from gx_doris_game_login_day_agg
where actday in ('2020-10-02','2020-10-03','2020-10-04')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'-- bitmap类型1
-- 2020-10-02 40475 32476 31688
-- 0.054s
select
'2020-10-02' actday,
intersect_count(user_id,actday,'2020-10-02') as active,
intersect_count(user_id,actday,'2020-10-02','2020-10-03') as retention1,
intersect_count(user_id,actday,'2020-10-02','2020-10-04') as retention2
from gx_doris_game_login_day_agg_bitmap
where actday in ('2020-10-02','2020-10-03','2020-10-04')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'-- bitmap类型2
-- 32476
SELECT
bitmap_count(bitmap_intersect(user_id)) as retention1
from
(
SELECT
actday,
bitmap_union(user_id) user_id
from gx_doris_game_login_day_agg_bitmap
where actday in ('2020-10-02','2020-10-03')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'
group by actday
)a-- bitmap类型3
-- 40475 32476 31688
select
bitmap_count(user01) as active,
bitmap_count(bitmap_and(user01,user02)) as retention1,
bitmap_count(bitmap_and(user01,user03)) as retention2
from
(
select
bitmap_union(case when actday='2020-10-02' then user_id end) as user01,
bitmap_union(case when actday='2020-10-03' then user_id end) as user02,
bitmap_union(case when actday='2020-10-04' then user_id end) as user03
from gx_doris_game_login_day_agg_bitmap
where actday in ('2020-10-02','2020-10-03','2020-10-04')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'
)aHLL
-- hll近似去重
-- 基本思想：a集合 + b集合 -（a ∪ b）= a ∩ b
-- hll 10543
SELECT
a+b-c as retention1
from
(
select
hll_union_agg(case when actday='2020-10-02' then HLL_HASH(user_id) end) as a,
hll_union_agg(case when actday='2020-10-03' then HLL_HASH(user_id) end) as b,
hll_union_agg(case when actday in ('2020-10-02','2020-10-03') then HLL_HASH(user_id) end) as c
from gx_doris_game_login_day_agg_1
where actday in ('2020-10-02','2020-10-03')
and appkey='mg_un_d2669937895c912d368426d84d1d8fd2'
)t四、总结
如果数据集的基数在百万、千万量级，使用 count distinct 即可；如果基数在亿级以上，并且需要精确去重，用Bitmap类型；如果可以接受近似去重，可以使用HLL类型。
原文链接：https://blog.csdn.net/weixin_43433596/article/details/119148601
作者：csdn、ベ斑驳
