redis7高级篇3 数据量亿级别的统计分析(hyperloglog,bitmap,geo)

一 亿级别统计分类

1.1 统计分类

1.聚合统计：统计多个集合聚合的结果，也就是多个集合之间交并差的统计。
2.排序统计：在需要展示最新列表，排行榜等场景时，如果数据更新频繁或者需要分页时，建议使用zset127.0.0.1:6379> zadd pl  111222 beijing 111223 tianjing 111333 shanghai
(integer) 3
127.0.0.1:6379> zrange pl 0 1
1) "beijing"
2) "tianjing"
127.0.0.1:6379> zrangebyscore  pl  111222  111233  limit 0 5
1) "beijing"
2) "tianjing"
127.0.0.1:6379> zrangebyscore  pl  111222  1114444  limit 0 5
1) "beijing"
2) "tianjing"
3) "shanghai"
127.0.0.1:6379> 

3.二值统计：集合中的元素只有0和1 ，钉钉打卡的场景，我们只用记录有签到的信息(1签到0没签到)
使用bitmap

4.基数统计：集合中不重复元素的总数，使用hyperloglog

127.0.0.1:6379> pfadd hp1  1 3 5 7 9
(integer) 1
127.0.0.1:6379> pfadd hp2  2 4 6 8 5
(integer) 1
127.0.0.1:6379> pfcount hp1
(integer) 5
127.0.0.1:6379> pfmerge distResult hp1 hp2
OK
127.0.0.1:6379> pfcount distResult
(integer) 9
127.0.0.1:6379> 

1.2 访问指标

uv：用户访问率，一般理解为客户端ip
pv：网页访问率
DAU: 日活跃用户量，登录或者使用了某个产品的用户数(去重复的登录的用户)
MAU：月活跃用户量

二 使用hyperloglog统计某宝uv的访问量案例

2.1 使用java代码实现

2.1.1 模拟用户不停访问某宝的数据

代码：

package com.ljf.redis.service.impl;

import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.util.Random;
import java.util.concurrent.TimeUnit;

/**
 * @auther zzyy
 * @create 2022-12-25 11:56
 */
@Service
@Slf4j
public class HyperLogLogService
{
    @Resource
    private RedisTemplate redisTemplate;

    /**
     * 模拟后台有用户点击淘宝首页www.taobao.com，每个用户来自不同的IP地址
     */
    @PostConstruct
    public void initIP()
    {
        new Thread(() -> {
            String ip = null;
            for (int i = 0; i < 20; i++) {
                Random random = new Random();
                ip = random.nextInt(256)+"."+
                        random.nextInt(256)+"."+
                        random.nextInt(256)+"."+
                        random.nextInt(256);

                Long hll = redisTemplate.opsForHyperLogLog().add("hll", ip);
                log.info("ip={},该IP地址访问首页的次数={}",ip,hll);
                //暂停3秒钟
                try { TimeUnit.SECONDS.sleep(3); } catch (InterruptedException e) { e.printStackTrace(); }
            }
        },"t1").start();
    }

    public long uv()
    {
        //PFCOUNT
        return redisTemplate.opsForHyperLogLog().size("hll");
    }

}

 2.1.2  查看代码编写

 代码

package com.ljf.redis.controller;


import com.ljf.redis.service.impl.HyperLogLogService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;

import javax.annotation.Resource;

/**
 * @auther zzyy
 * @create 2022-12-25 11:56
 */
@Api(tags = "淘宝亿级UV的Redis统计方案")
@RestController
@Slf4j
public class HyperLogLogController
{
    @Resource
    HyperLogLogService hyperLogLogService;

    @ApiOperation("获得IP去重复后的UV统计访问量")
    @RequestMapping(value = "/uv",method = RequestMethod.GET)
    public long uv()
    {
        return hyperLogLogService.uv();
    }
}

2.2  启动redis服务

1.启动redis服务

[root@localhost ~]# redis-server /myredis/redis.conf 
[root@localhost ~]# redis-cli -a 123456 -p 6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> keys *

2.关闭防火墙

 2.3 查看

1.页面访问

2.redis查看

三 使用GEO获取指定位置附近的坐标

3.1 java实现

1.controller

package com.ljf.redis.controller;


import com.ljf.redis.service.impl.GeoService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.geo.Distance;
import org.springframework.data.geo.GeoResults;
import org.springframework.data.geo.Point;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;

import javax.annotation.Resource;

/**
 * @auther zzyy
 * @create 2022-12-25 12:12
 */
@Api(tags = "美团地图位置附近的酒店推送GEO")
@RestController
@Slf4j
public class GeoController
{
    @Resource
    private GeoService geoService;

    @ApiOperation("添加坐标geoadd")
    @RequestMapping(value = "/geoadd",method = RequestMethod.GET)
    public String geoAdd()
    {
        return geoService.geoAdd();
    }

    @ApiOperation("获取经纬度坐标geopos")
    @RequestMapping(value = "/geopos",method = RequestMethod.GET)
    public Point position(String member)
    {
        return geoService.position(member);
    }

    @ApiOperation("获取经纬度生成的base32编码值geohash")
    @RequestMapping(value = "/geohash",method = RequestMethod.GET)
    public String hash(String member)
    {
        return geoService.hash(member);
    }

    @ApiOperation("获取两个给定位置之间的距离")
    @RequestMapping(value = "/geodist",method = RequestMethod.GET)
    public Distance distance(String member1, String member2)
    {
        return geoService.distance(member1,member2);
    }

    @ApiOperation("通过经度纬度查找北京王府井附近的")
    @RequestMapping(value = "/georadius",method = RequestMethod.GET)
    public GeoResults radiusByxy()
    {
        return geoService.radiusByxy();
    }

    @ApiOperation("通过地方查找附近,本例写死天安门作为地址,作为家庭作业")
    @RequestMapping(value = "/georadiusByMember",method = RequestMethod.GET)
    public GeoResults radiusByMember()
    {
        return geoService.radiusByMember();
    }

}

2.servie

package com.ljf.redis.service.impl;

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.geo.*;
import org.springframework.data.redis.connection.RedisGeoCommands;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * @auther zzyy
 * @create 2022-12-25 12:11
 */
@Service
@Slf4j
public class GeoService
{
    public static final String CITY ="city";
    @Autowired
    private RedisTemplate redisTemplate;

    public String geoAdd()
    {
        Map map = new HashMap<>();
        map.put("天安门",new Point(116.403963,39.915119));
        map.put("故宫",new Point(116.403414,39.924091));
        map.put("长城",new Point(116.024067,40.362639));

        redisTemplate.opsForGeo().add(CITY,map);

        return map.toString();
    }

    public Point position(String member) {
        //获取经纬度坐标
        List list = redisTemplate.opsForGeo().position(CITY, member);
        System.out.println("获取经纬度....");
        return list.get(0);
    }

    public String hash(String member) {
        //geohash算法生成的base32编码值
        List list = redisTemplate.opsForGeo().hash(CITY, member);
        return list.get(0);
    }

    public Distance distance(String member1, String member2) {
        //获取两个给定位置之间的距离
        Distance distance = redisTemplate.opsForGeo().distance(CITY, member1, member2,
                RedisGeoCommands.DistanceUnit.KILOMETERS);
        return distance;
    }
    public GeoResults radiusByxy() {
        //通过经度，纬度查找附近的,北京王府井位置116.418017,39.914402
        Circle circle = new Circle(116.418017, 39.914402, Metrics.KILOMETERS.getMultiplier());
        // 返回50条
        RedisGeoCommands.GeoRadiusCommandArgs args = RedisGeoCommands.GeoRadiusCommandArgs.newGeoRadiusArgs().includeDistance().includeCoordinates().sortDescending().limit(50);

        GeoResults> geoResults = redisTemplate.opsForGeo().radius(CITY, circle, args);

        return geoResults;
    }
    public GeoResults radiusByMember() {
        //通过地方查找附近
        return null;
    }
}

3.2 测试

1.添加

 

2.获取坐标

3.计算距离

 四 使用bitmap实现统计

4.1 常用命令

应用场景：

连续签到打卡
最近一周的活跃用户
统计指定用户一年之中的登录天数

1.添加数据

127.0.0.1:6379> setbit u1:20230822 0 1
0
127.0.0.1:6379> setbit u2:20230822 0 1
0

2.获取数据
127.0.0.1:6379> getbit u1:20230822
ERR wrong number of arguments for 'getbit' command

127.0.0.1:6379> getbit u1:20230822 3
0
127.0.0.1:6379> getbit u1:20230822 0
1

3.统计数目

127.0.0.1:6379> bitcount u1:20230822 0 30
1
127.0.0.1:6379>