使用EasyExcel进行百万数据文件导出思路
只做演示，所以都写在Controller里面了；
1：添加EasyExcel的依赖
com.alibaba
easyexcel
3.0.5
javax.servlet
servlet.api
org.apache.poi
poi
org.apache.poi
poi-ooxml
org.apache.poi
poi-ooxml-schemas
1：定义线程池ThreadPoolConfig
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadPoolExecutor;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import com.ahzx.common.utils.Threads;
/**
* 线程池配置
*
**/
@Configuration
public class ThreadPoolConfig {
// 核心线程池大小
private int corePoolSize = 50;
// 最大可创建的线程数
private int maxPoolSize = 200;
// 队列最大长度
private int queueCapacity = 1000;
// 线程池维护线程所允许的空闲时间
private int keepAliveSeconds = 300;
@Bean(name = "threadPoolTaskExecutor")
public ThreadPoolTaskExecutor threadPoolTaskExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setMaxPoolSize(maxPoolSize);
executor.setCorePoolSize(corePoolSize);
executor.setQueueCapacity(queueCapacity);
executor.setKeepAliveSeconds(keepAliveSeconds);
// 线程池对拒绝任务(无线程可用)的处理策略
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
return executor;
}
}
1：定义EasyExcel导出的实体类
@Data
public class FmrInfoDtoEeayExcel {
@ExcelProperty(value = "ID")
private String homeNo;
@ExcelProperty(value = "等级")
private String rank;
@ExcelProperty(value = "分数")
private Integer score;
@ExcelProperty(value = "状态")
private String scoreStatus;
//多余的一些字段...
}
2：定义Controller
@PostMapping("/exportAll")
public synchronized void exportAll(HttpServletResponse response) {
log.info("sql分页导出excel....");
//用于计算方法运行时间，为了方便直接用Data
Date start = new Date();
//定义easyExcel导出的对象
ExcelWriter excelWriter = null;
//导出的文件名
String fileName = "/tmp/导出的excel文件名称.xlsx";
//主要进行处理的方法，用于返回需要的集合
List listFaram = this.multiThreadList();
try{
excelWriter = EasyExcel.write(fileName,FmrInfoDtoEeayExcel.class).build();
WriteSheet writeSheet = EasyExcel.writerSheet("sheet1").build();
excelWriter.write(listFaram,writeSheet);
Date end = new Date();
String datePoor = DateUtils.getDatePoor(end, start);
log.info("导出耗时：" + datePoor);
} catch(Exception e){
logger.debug("文件导出报错，{}",e.getMessage());
}finally {
if(excelWriter != null ){
excelWriter.finish();
}
}
} 3：定于multiThreadList方法
//定义PAGENUM 也就是每个线程查询的页数
private static final int PAGENUM = 5000;
private List multiThreadList(){
//定义多线程的任务
List>> tasks = new ArrayList<>();
//定义导出的集合
List fmrInfoDtoEeayExcelList = new ArrayList<>();
//查询要导出表的总数
long count = tbFmrInfoService.selectTbFmrInfoCount();
//计算开启的线程数
int loopNum = new Double(Math.ceil((double)count / PAGENUM )).intValue();
logger.info("多线程查询，总数：{}，开启线程数：{}",count,loopNum);
executeTask(tasks,loopNum,count);
for(FutureTask> task : tasks){
try{
fmrInfoDtoEeayExcelList.addAll(task.get());
}catch(Exception e){
logger.debug("出错了：{}",e.getMessage());
}
}
return fmrInfoDtoEeayExcelList;
} 4：定义executeTask方法
private void executeTask(List>> tasks, int loopNum, long count) {
for (int i = 0; i < loopNum; i++) {
Map map = new HashMap<>();
map.put("offset", i * PAGENUM);
if( i == loopNum -1 ){
map.put("limit",count - PAGENUM * i);
}else{
map.put("limit",PAGENUM);
}
FutureTask> task = new FutureTask<>(new listThread(map));
logger.info("开始查询第{}条开始的{}条记录",i * PAGENUM, PAGENUM);
//new Thread(task).start();
threadPool.threadPoolTaskExecutor().execute(task);
tasks.add(task);
}
} 5：定于多线程处理的内部类
//因为数据不在同一个表里面，所以查多张表，进行处理
private class listThread implements Callable> {
private final Map map;
public listThread(Map map) {
this.map = map;
}
@Override
public List call() throws Exception {
//分页查询出5000条主表数据
List fmrInfoDtos = tbFmrInfoMapper.selectTbFmrInfoRankListPage(map);
//用stream流处理编号，提取出编号的集合
List homeNoList = fmrInfoDtos.stream().map(s -> s.getHomeNo()).collect(Collectors.toList());
//用编号集合去另外一个表里面查这个对象的分数
List gradeScore = gradeRankMapper.selectScoreByFrmIdList(homeNoList);
//查出分数的集合，将两个集合根据一定的要求进行字段处理
fmrInfoDtos.forEach(e -> {
if (StringUtils.isNotNull(e.getRank())) {
e.setScoreStatus("已评分");
} else {
e.setScoreStatus("未评分");
}
//处理身份证号码
if (StringUtils.isNotEmpty(e.getCardNo())) {
e.setBirthDay(StringUtils.substring(e.getCardNo(), 6, 14));
}
//两者id相同，则将查出来的分数进行赋值处理
gradeScore.forEach(n -> {
if (e.getHomeNo().equals(n.getObjectNo())) {
e.setScore(n.getScore().intValue());
}
});
});
return fmrInfoDtos;
}
} 6：对应的mapper文件
//tbFmrInfoService.selectTbFmrInfoCount();
public Long selectTbFmrInfoCount() {
return tbFmrInfoMapper.selectTbFmrInfoCount();
}
// tbFmrInfoMapper.selectTbFmrInfoRankListPage(map);
//gradeRankMapper.selectScoreByFrmIdList(homeNoList);此处使用in查询
7：至此多线程文件导出完毕，其中线程池的那边和sql还有优化空间，整个数据大概40w，处理完毕大概20s左右，我感觉还是很慢，写的还是有问题，仅供参考，如果有更好的方法希望可以多加讨论