EasyExcel3.0.5 加快大数据查询速度，查询性能优化
文章目录
- 前言
- 技术栈和代码结构
- 关键的导出监听器
- Service 实现
- 测试
前言
语雀社区写的 easyExcel 确实是目前市面excel 导入导出性能最好的框架，使用简便。社区指导清晰，不需要那么繁杂的代码。
前面写了一篇 EasyExcel3.0.5 解决大数据导入导出，防止OOM。后来，在工作中我让一个同事参照我的去实现，他请教了他们组长，给了点查询优化建议，确实值得改善，这给了我一些启发，今天分享一下。
影响速度很关键的是网络，其次是，数据表的索引优化，第三是我们自己写查询语句时做优化，话不多说，上干货。
技术栈和代码结构
沿用上一篇的技术栈和代码结构，详情请移步 EasyExcel3.0.5 解决大数据导入导出，防止OOM 。
关键的导出监听器
ExportListener.java
package cn.com.easyExcel.excel.listener;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.springframework.util.CollectionUtils;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.URLEncoder;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
public class ExportListener<T> {
private BaseMapper<T> baseMapper;
public ExportListener(BaseMapper<T> baseMapper) {
this.baseMapper = baseMapper;
}
private static final String DATA_FORMAT = "yyyy-MM-dd-HH-mm-ss";
private static final String CHARACTER_UTF_8 = "UTF-8";
private static final String CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String CONTENT_DISPOSITION = "Content-Disposition";
private static final String CACHE_CONTROL = "Cache-Control";
private static final String NO_STORE = "no-store";
private static final String MAX_AGE = "max-age=0";
private static final int PAGE_SIZE = 10000;
public void exportExcel(HttpServletResponse response, String sheetName, Class<T> pojoClass,
LambdaQueryWrapper<T> queryWrapper) throws IOException {
ServletOutputStream out = getServletOutputStream(response, sheetName);
// 这里 需要指定写用哪个class去写
ExcelWriter excelWriter = EasyExcel.write(out, pojoClass).build();
// 这里注意 如果同一个sheet只要创建一次
WriteSheet writeSheet = EasyExcel.writerSheet(sheetName).build();
int totalCount = Math.toIntExact(baseMapper.selectCount(queryWrapper));
int pageNumber = (int) Math.ceil((double) totalCount / (double) PAGE_SIZE); //分页条数看情况
// 去调用写入,根据数据库分页的总的页数来
for (int i = 1; i <= pageNumber; i++) {
//先定义一个空集合每次循环使他变成null减少内存的占用
List<T> recordList = new ArrayList<>();
Page<T> page = new Page<>(i, PAGE_SIZE);
Page<T> pojoIPage = baseMapper.selectPage(page, queryWrapper);
recordList = pojoIPage.getRecords();
excelWriter.write(recordList , writeSheet);
recordList.clear();
}
// 千万别忘记finish 会帮忙关闭流
excelWriter.finish();
out.flush();
}
/**
* 查询优化的方法
*/
public void exportNoQueryCount(HttpServletResponse response, String sheetName, Class<T> pojoClass,
LambdaQueryWrapper<T> queryWrapper) throws IOException {
response.setContentType(CONTENT_TYPE);
response.setCharacterEncoding(CHARACTER_UTF_8);
// URLEncoder.encode可以防止中文乱码
String fileName = URLEncoder.encode(sheetName, CHARACTER_UTF_8).replaceAll("\\+", "%20");
response.setHeader(CONTENT_DISPOSITION, "attachment;filename*=utf-8''" + fileName + ".xlsx");
ExcelWriter excelWriter = EasyExcel.write(response.getOutputStream(), pojoClass).build();
// 这里注意 如果同一个sheet只要创建一次
WriteSheet writeSheet = EasyExcel.writerSheet(sheetName).build();
int startIndex = 1;
while (true){
int startParam =(startIndex-1) * PAGE_SIZE;
int pageNumber = (int) Math.ceil((double) startParam / (double) PAGE_SIZE+1);
Page<T> page = new Page<>(pageNumber, PAGE_SIZE, false);
Page<T> pojoIPage = baseMapper.selectPage(page, queryWrapper);
List<T> recordList = pojoIPage.getRecords();
if (CollectionUtils.isEmpty(recordList)) {
break;
}
startIndex++;
excelWriter.write(recordList , writeSheet);
}
// 千万别忘记finish 会帮忙关闭流
excelWriter.finish();
}
public ServletOutputStream getServletOutputStream(HttpServletResponse response, String sheetName) throws IOException {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DATA_FORMAT);
String nowTime = formatter.format(LocalDateTime.now());
String fileName = sheetName.concat("_").concat(nowTime).concat(".xlsx");
response.setContentType(CONTENT_TYPE);
//设置字符集为utf-8
response.setCharacterEncoding(CHARACTER_UTF_8);
//用postman测正常，浏览器多了filename_=utf-8等字样
response.setHeader(CONTENT_DISPOSITION,
"attachment;filename=" + URLEncoder.encode(fileName, CHARACTER_UTF_8)
+ ";filename*=utf-8''" + URLEncoder.encode(fileName, CHARACTER_UTF_8));
//postman测会乱码，但浏览器下载就正常
// response.setHeader(CONTENT_DISPOSITION,
// "attachment;filename=" + URLEncoder.encode(fileName, "UTF-8"));
//发送一个报头，告诉浏览器当前页面不进行缓存，每次访问的时间必须从服务器上读取最新的数据
response.setHeader(CACHE_CONTROL, NO_STORE);
response.addHeader(CACHE_CONTROL, MAX_AGE);
return response.getOutputStream();
}
}
Service 实现
package cn.com.easyExcel.service.impl;
import cn.com.easyExcel.excel.listener.ExportListener;
import cn.com.easyExcel.excel.listener.ImportListener;
import cn.com.easyExcel.mapper.EmployeeMapper;
import cn.com.easyExcel.pojo.EmployeeExporter;
import cn.com.easyExcel.service.EmployeeService;
import com.alibaba.excel.EasyExcel;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class EmployeeServiceImpl implements EmployeeService {
@Autowired
private EmployeeMapper employeeMapper;
@Override
public void initData() {
long beforeTime = System.currentTimeMillis();
List<EmployeeExporter> employees = new ArrayList<EmployeeExporter>();
for (int i = 0; i < 200000; i++) {
EmployeeExporter employee = new EmployeeExporter();
employee.setUserName(getRandomName());
employee.setGender(getRandomGender());
employee.setAge(getRandomAge());
employee.setMaritalStatus(getRandomGender());
employee.setEducation(getRandomEducation());
employee.setMobile("18866998888");
employee.setDepartmentName(getRandomDP());
employee.setNationalArea("中国");
employee.setCity("深圳");
employees.add(employee);
if(employees.size() % 1000 == 0){
employeeMapper.batchInsert(employees);
employees.clear();
}
}
long afterTime = System.currentTimeMillis();
log.info("耗时:{}", afterTime - beforeTime);
}
@Override
public void importExcel(MultipartFile file) throws IOException {
long beforeTime = System.currentTimeMillis();
EasyExcel.read(file.getInputStream(),
EmployeeExporter.class,
new ImportListener(employeeMapper)).sheet().headRowNumber(1).doRead();
long afterTime = System.currentTimeMillis();
log.info("耗时:{}", afterTime - beforeTime);
}
@Override
public void exportExcel(HttpServletResponse response) throws IOException {
long beforeTime = System.currentTimeMillis();
LambdaQueryWrapper<EmployeeExporter> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.gt(EmployeeExporter::getAge, 22);
queryWrapper.between(EmployeeExporter::getEducation, "1", "3");
new ExportListener<>(employeeMapper).
exportExcel(response, "员工信息", EmployeeExporter.class,
queryWrapper);
long afterTime = System.currentTimeMillis();
log.info("耗时:{}", afterTime - beforeTime);
}
@Override
public void exportExcelNoQueryCount(HttpServletResponse response) throws IOException {
long beforeTime = System.currentTimeMillis();
LambdaQueryWrapper<EmployeeExporter> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.between(EmployeeExporter::getAge, "25", "30");
new ExportListener<>(employeeMapper).
exportNoQueryCount(response, "员工信息", EmployeeExporter.class,
queryWrapper);
long afterTime = System.currentTimeMillis();
log.info("导出查询耗时33:{}", afterTime - beforeTime);
}
/**
* 随机取名字
* @return
*/
public String getRandomName(){
String[] doc = {"朝歌晚酒", "都怪时光太动听", "笑我孤陋", "水墨青花","时光清浅", "草帽撸夫", "江山如画",
"热度不够", "盏茶浅抿", "把酒临风", "且听风吟", "梦忆笙歌", "倾城月下", "清风墨竹", "自愈心暖", "几许轻唱",
"平凡之路", "半夏倾城", "南栀倾寒", "孤君独战", "温酒杯暖", "眉目亦如画", "旧雪烹茶", "律断华章", "清酒暖风",
"清羽墨安", "一夕夙愿", "南顾春衫", "和云相伴", "夕颜若雪", "时城旧巷", "梦屿千寻", "故港笑别", "水袖萦香",
"秋水墨凉", "海棠花瘦", "千城暮雪", "华灯初上", "一纸枕书", "剑断青丝", "风烟影月", "日月星辰", "浅喜深爱"};
int index = (int) (Math.random() * doc.length);
return doc[index];
}
/**
* 性别随机
* @return
*/
public String getRandomGender(){
String[] doc = {"0", "1"};
int index = (int) (Math.random() * doc.length);
return doc[index];
}
/**
* 年龄随机
* @return
*/
public int getRandomAge(){
int[] doc = {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30};
int index = (int) (Math.random() * doc.length);
return doc[index];
}
public String getRandomEducation(){
String[] doc = {"0", "1", "2", "3"};
int index = (int) (Math.random() * doc.length);
return doc[index];
}
public String getRandomDP(){
String[] doc = {"行政部", "财务部", "技术部", "市场部", "公关部"};
int index = (int) (Math.random() * doc.length);
return doc[index];
}
}
测试
我用 20万数据区测试，摈除家庭网速一般的因素，用第一个没有改造的方法去导出，平均耗时70+ 秒，用第二种改造的方法平均耗时 30秒左右。 这还是在我的网速一般的情况，而且我没有做什么索引优化。
exportExcel 方法测试的结果：
exportNoQueryCount 方法测试的结果：
好了，分享结束。 不整噱头，不标题党（什么史上最强/最全/最好，看这篇足够了……，标题党的内容八成都是很水的抄袭），认真分享，认真探索，觉得有用，欢迎留言。