目录
1.Redis 高级数据类型
2.网站数据统计
2.1 业务层
2.2 表现层
2.2.1 记录数据
2.2.2 查看数据
HyperLogLog:采用一种基数算法,用于完成独立总数的统计;占据空间小,无论统计多少个数据,只占12K的内存空间;不精确的统计算法,标准误差为 0.81%
Bitmap:不是一种独立的数据结构,实际上就是字符串;支持按位存取数据,可以将其看成是 byte 数组;适合存储索大量的连续的数据的布尔值
统计 20万个重复数据的独立总数
// 统计20万个重复数据的独立总数.
@Test
public void testHyperLogLog() {
String redisKey = "test:hll:01";
for (int i = 1; i <= 100000; i++) {
redisTemplate.opsForHyperLogLog().add(redisKey, i);
}
//再次循环 10万次
for (int i = 1; i <= 100000; i++) {
int r = (int) (Math.random() * 100000 + 1);
redisTemplate.opsForHyperLogLog().add(redisKey, r);
}
long size = redisTemplate.opsForHyperLogLog().size(redisKey);//统计去重数据的数量
System.out.println(size);
}
将3组数据合并,再统计合并后的重复数据的独立总数
@Test
public void testHyperLogLogUnion() {
String redisKey2 = "test:hll:02";
for (int i = 1; i <= 10000; i++) {
redisTemplate.opsForHyperLogLog().add(redisKey2, i);
}
String redisKey3 = "test:hll:03";
for (int i = 5001; i <= 15000; i++) {
redisTemplate.opsForHyperLogLog().add(redisKey3, i);
}
String redisKey4 = "test:hll:04";
for (int i = 10001; i <= 20000; i++) {
redisTemplate.opsForHyperLogLog().add(redisKey4, i);
}
String unionKey = "test:hll:union";
redisTemplate.opsForHyperLogLog().union(unionKey, redisKey2, redisKey3, redisKey4);
long size = redisTemplate.opsForHyperLogLog().size(unionKey);
System.out.println(size);
}
统计一组数据的布尔值
@Test
public void testBitMap() {
String redisKey = "test:bm:01";
// 记录
redisTemplate.opsForValue().setBit(redisKey, 1, true);
redisTemplate.opsForValue().setBit(redisKey, 4, true);
redisTemplate.opsForValue().setBit(redisKey, 7, true);
// 查询
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 0));
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 1));
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 2));
// 统计
Object obj = redisTemplate.execute(new RedisCallback() {
@Override
public Object doInRedis(RedisConnection connection) throws DataAccessException {
return connection.bitCount(redisKey.getBytes());
}
});
System.out.println(obj);
}
统计3组数据的布尔值, 并对这3组数据做OR运算
@Test
public void testBitMapOperation() {
String redisKey2 = "test:bm:02";
redisTemplate.opsForValue().setBit(redisKey2, 0, true);
redisTemplate.opsForValue().setBit(redisKey2, 1, true);
redisTemplate.opsForValue().setBit(redisKey2, 2, true);
String redisKey3 = "test:bm:03";
redisTemplate.opsForValue().setBit(redisKey3, 2, true);
redisTemplate.opsForValue().setBit(redisKey3, 3, true);
redisTemplate.opsForValue().setBit(redisKey3, 4, true);
String redisKey4 = "test:bm:04";
redisTemplate.opsForValue().setBit(redisKey4, 4, true);
redisTemplate.opsForValue().setBit(redisKey4, 5, true);
redisTemplate.opsForValue().setBit(redisKey4, 6, true);
String redisKey = "test:bm:or";
Object obj = redisTemplate.execute(new RedisCallback() {
@Override
public Object doInRedis(RedisConnection connection) throws DataAccessException {
connection.bitOp(RedisStringCommands.BitOperation.OR,
redisKey.getBytes(), redisKey2.getBytes(), redisKey3.getBytes(), redisKey4.getBytes());
return connection.bitCount(redisKey.getBytes());
}
});
System.out.println(obj);
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 0));
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 1));
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 2));
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 3));
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 4));
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 5));
System.out.println(redisTemplate.opsForValue().getBit(redisKey, 6));
}
使用 Redis,定义 RedisKey,打开 RedisKeyUtil 类添加:
//UV(Unique Visitor):独立访问
private static final String PREFIX_UV = "uv";
//DAU(Daily Active User):日活跃用户
private static final String PREFIX_DAU = "dau";
//单日UV
public static String getUVKey(String date) {
return PREFIX_UV + SPLIT + date;
}
//区间UV
public static String getUVKey(String startDate, String endDate) {
return PREFIX_DAU + SPLIT + startDate + SPLIT + endDate;
}
// 单日活跃用户
public static String getDAUKey(String date) {
return PREFIX_DAU + SPLIT + date;
}
// 区间活跃用户
public static String getDAUKey(String startDate, String endDate) {
return PREFIX_DAU + SPLIT + startDate + SPLIT + endDate;
}
在 service 包下新建 DataService 类:
package com.example.demo.service;
import com.example.demo.util.RedisKeyUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DataAccessException;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.RedisStringCommands;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
/**
* 网站数据统计:UV、DAU
*/
@Service
public class DataService {
@Autowired
private RedisTemplate redisTemplate;
//在统计的时候,需要使用到日期(格式化成年月日的形式),实例化一个 SimpleDateFormat
private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd");
// 将指定的IP计入UV
public void recordUV(String ip) {
//得到 key记录到 Redis 中
String redisKey = RedisKeyUtil.getUVKey(df.format(new Date()));
redisTemplate.opsForHyperLogLog().add(redisKey, ip);
}
// 统计指定日期范围内的UV
public long calculateUV(Date start, Date end) {
if (start == null || end == null) {
throw new IllegalArgumentException("参数不能为空!");
}
// 整理该日期范围内的key
//把范围内每一天的 key 做一个合并得到某一组的 key,封装成集合;
// 遍历日期,需要对日期做运算,实例化Calender,包含开始日期做遍历。遍历完成之后合并数据并且返回统计的结果
List keyList = new ArrayList<>();
Calendar calendar = Calendar.getInstance();
calendar.setTime(start);
while (!calendar.getTime().after(end)) {
String key = RedisKeyUtil.getUVKey(df.format(calendar.getTime()));
keyList.add(key);
calendar.add(Calendar.DATE, 1);
}
// 合并这些数据
String redisKey = RedisKeyUtil.getUVKey(df.format(start), df.format(end));
redisTemplate.opsForHyperLogLog().union(redisKey, keyList.toArray());
// 返回统计的结果
return redisTemplate.opsForHyperLogLog().size(redisKey);
}
// 将指定用户计入DAU
public void recordDAU(int userId) {
String redisKey = RedisKeyUtil.getDAUKey(df.format(new Date()));
redisTemplate.opsForValue().setBit(redisKey, userId, true);
}
// 统计指定日期范围内的DAU
public long calculateDAU(Date start, Date end) {
if (start == null || end == null) {
throw new IllegalArgumentException("参数不能为空!");
}
// 整理该日期范围内的key
List keyList = new ArrayList<>();
Calendar calendar = Calendar.getInstance();
calendar.setTime(start);
while (!calendar.getTime().after(end)) {
String key = RedisKeyUtil.getDAUKey(df.format(calendar.getTime()));
keyList.add(key.getBytes());
calendar.add(Calendar.DATE, 1);
}
// 进行OR运算
return (long) redisTemplate.execute(new RedisCallback() {
@Override
public Object doInRedis(RedisConnection connection) throws DataAccessException {
String redisKey = RedisKeyUtil.getDAUKey(df.format(start), df.format(end));
connection.bitOp(RedisStringCommands.BitOperation.OR,
redisKey.getBytes(), keyList.toArray(new byte[0][0]));
return connection.bitCount(redisKey.getBytes());
}
});
}
}
什么时候记录数据(拦截器)、查看数据
在 controller 包下的 interceptor 包下新建 DataInterceptor 类:
package com.example.demo.controller.interceptor;
import com.example.demo.entity.User;
import com.example.demo.service.DataService;
import com.example.demo.util.HostHolder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.HandlerInterceptor;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@Component
public class DataInterceptor implements HandlerInterceptor {
@Autowired
private DataService dataService;
@Autowired
private HostHolder hostHolder;
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
// 统计UV
String ip = request.getRemoteHost();
dataService.recordUV(ip);
// 统计DAU
User user = hostHolder.getUser();
if (user != null) {
dataService.recordDAU(user.getId());
}
return true;
}
}
在 WebMvcConfig 类中设置拦截器:
@Autowired
private MessageInterceptor messageInterceptor;
registry.addInterceptor(dataInterceptor)
.excludePathPatterns("/**/*.css", "/**/*.js", "/**/*.png", "/**/*.jpg", "/**/*.jpeg");
在 controller 类包下新建 DataController 类:
package com.example.demo.controller;
import com.example.demo.service.DataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import java.util.Date;
/**
* 网站数据统计:UV、DAU
*/
@Controller
public class DataController {
@Autowired
private DataService dataService;
// 统计页面
@RequestMapping(path = "/data", method = {RequestMethod.GET, RequestMethod.POST})
public String getDataPage() {
return "/site/admin/data";
}
// 统计网站UV
@RequestMapping(path = "/data/uv", method = RequestMethod.POST)
public String getUV(@DateTimeFormat(pattern = "yyyy-MM-dd") Date start,
@DateTimeFormat(pattern = "yyyy-MM-dd") Date end, Model model) {
long uv = dataService.calculateUV(start, end);
model.addAttribute("uvResult", uv);
model.addAttribute("uvStartDate", start);
model.addAttribute("uvEndDate", end);
return "forward:/data";
}
// 统计活跃用户
@RequestMapping(path = "/data/dau", method = RequestMethod.POST)
public String getDAU(@DateTimeFormat(pattern = "yyyy-MM-dd") Date start,
@DateTimeFormat(pattern = "yyyy-MM-dd") Date end, Model model) {
long dau = dataService.calculateDAU(start, end);
model.addAttribute("dauResult", dau);
model.addAttribute("dauStartDate", start);
model.addAttribute("dauEndDate", end);
return "forward:/data";
}
}
最后处理 data.html