POI高效百万级别Excel导出-1min轻松解决问题
思路：在数据量达到百万级别的情况下，如果一次性读取并写入Excel会比较慢，如果将数据写入同一个Excel的多个Sheet，并分别通过多个线程分别写Sheet，效率将会提高很多。
经测试：100w行，每行16列的数据导出大概60s到70s之间。
使用工具：
// 总记录数
Integer totalRowCount =ipccDao.getSearchSize("customer.selectExportCustNum", params);
// 导出EXCEL文件名称
String filaName = "客户信息";
// 标题
String[] titles = {"客户名称", "客户名称", "联系人", "联系方式", "负责人",
"分配时间", "客户意向等级", "最后跟进时间", "通话状态","跟进阶段",
"通话轮次","最近外呼任务","客户来源","创建时间","客户关注点","通话时长"};
// 开始导入
PoiUtil.exportExcelToWebsite(StrutsUtils.getResponse(), totalRowCount, filaName, titles,
new WriteExcelDataDelegated() {
@Override
public void writeExcelData(SXSSFSheet eachSheet, Integer startRowCount, Integer endRowCount,
Integer currentPage, Integer pageSize) throws Exception {
int startRow = (currentPage - 1) * pageSize; // 分页开始行号
params.put("limit", " LIMIT " + startRow + ", " + pageSize);
//List> exportCust = customerService.exportCust(params);
List> exportCust = ipccDao.getSearchList("customer.selectCustomer", params);
if (!CollectionUtils.isEmpty(exportCust)) {
for (int i = startRowCount; i <= endRowCount; i++) {
SXSSFRow eachDataRow = eachSheet.createRow(i);
if ((i - startRowCount) < exportCust.size()) {
Map customer = exportCust.get(i - startRowCount);
eachDataRow.createCell(0).setCellValue(
customer.get("name") != null ? customer.get("name") + "" : "");
eachDataRow.createCell(1).setCellValue(customer.get("customer_nbr") != null? customer.get("customer_nbr") + "": "");
eachDataRow.createCell(2).setCellValue(customer.get("contacts_name") != null? customer.get("contacts_name") + "": "");
eachDataRow.createCell(3).setCellValue(customer.get("contacts_nbr")!=null?customer.get("contacts_nbr")+ "":"");
eachDataRow.createCell(4).setCellValue(customer.get("leader")!=null?customer.get("leader") + "":"");
eachDataRow.createCell(5).setCellValue(customer.get("distribution_date")!=null?customer.get("distribution_date") + "":"");
eachDataRow.createCell(6).setCellValue(customer.get("cust_type")!=null?customer.get("cust_type") + "":"");
eachDataRow.createCell(7).setCellValue(customer.get("last_contact_date")!=null?customer.get("last_contact_date") + "":"");
changeCallStatusToMsg(customer);
eachDataRow.createCell(8).setCellValue(customer.get("status") + "");
changeContactGradation(customer);
eachDataRow.createCell(9).setCellValue(customer.get("contact_gradation") + "");
eachDataRow.createCell(10).setCellValue(customer.get("talk_times")!=null?customer.get("talk_times") + "":"");
eachDataRow.createCell(11).setCellValue(customer.get("task_name")!=null?customer.get("task_name") + "":"");
eachDataRow.createCell(12).setCellValue(customer.get("finder") !=null?customer.get("finder") + "":"");
eachDataRow.createCell(13).setCellValue(customer.get("create_date")!=null?customer.get("create_date") + "":"");
eachDataRow.createCell(14).setCellValue(customer.get("result_label")!=null?customer.get("result_label") + "":"");
eachDataRow.createCell(15).setCellValue(customer.get("tel_time")!=null?customer.get("tel_time") + "":"");
}
}
}
}
}); POIUtil.java工具类
package com.admin.util;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import javax.servlet.http.HttpServletResponse;
import org.apache.poi.xssf.streaming.SXSSFCell;
import org.apache.poi.xssf.streaming.SXSSFRow;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.admin.task.ExportExcelCallable;
/**
* @author qjwyss
* @date 2018/9/18
* @description POI导出工具类
*/
public class PoiUtil {
/**
* 每个sheet存储的记录数10W
*/
public static Integer PER_SHEET_ROW_COUNT;
/**
* 每次向EXCEL写入的记录数(查询每页数据大小) 5W
*/
public static Integer PER_WRITE_ROW_COUNT;
static{
PER_SHEET_ROW_COUNT = SysConfig.getInstance().getPropertyInt("PER_SHEET_ROW_COUNT",200000);
PER_WRITE_ROW_COUNT = SysConfig.getInstance().getPropertyInt("PER_WRITE_ROW_COUNT",100000);
}
/**
* 每个sheet的写入次数 2
*/
public static final Integer PER_SHEET_WRITE_COUNT = PER_SHEET_ROW_COUNT / PER_WRITE_ROW_COUNT;
private final static Logger logger = LoggerFactory.getLogger(PoiUtil.class);
/**
* 初始化EXCEL(sheet个数和标题)
*
* @param totalRowCount 总记录数
* @param titles 标题集合
* @return XSSFWorkbook对象
*/
public static SXSSFWorkbook initExcel(Integer totalRowCount, String[] titles) {
logger.info("PER_SHEET_ROW_COUNT：{}，PER_WRITE_ROW_COUNT：{}",PER_SHEET_ROW_COUNT,PER_WRITE_ROW_COUNT);
// 在内存当中保持 100 行 , 超过的数据放到硬盘中在内存当中保持 100 行 , 超过的数据放到硬盘中
SXSSFWorkbook wb = new SXSSFWorkbook(100);
Integer sheetCount = ((totalRowCount % PER_SHEET_ROW_COUNT == 0) ?
(totalRowCount / PER_SHEET_ROW_COUNT) : (totalRowCount / PER_SHEET_ROW_COUNT + 1));
// 根据总记录数创建sheet并分配标题
for (int i = 0; i < sheetCount; i++) {
SXSSFSheet sheet = wb.createSheet("sheet" + (i + 1));
sheet.setRandomAccessWindowSize(-1);
SXSSFRow headRow = sheet.createRow(0);
for (int j = 0; j < titles.length; j++) {
SXSSFCell headRowCell = headRow.createCell(j);
headRowCell.setCellValue(titles[j]);
}
}
return wb;
}
/**
* 下载EXCEL到本地指定的文件夹
*
* @param wb EXCEL对象SXSSFWorkbook
* @param exportPath 导出路径
*/
public static void downLoadExcelToLocalPath(SXSSFWorkbook wb, String exportPath) {
FileOutputStream fops = null;
try {
fops = new FileOutputStream(exportPath);
wb.write(fops);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != wb) {
try {
wb.dispose();
} catch (Exception e) {
e.printStackTrace();
}
}
if (null != fops) {
try {
fops.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
/**
* 下载EXCEL到浏览器
*
* @param wb EXCEL对象XSSFWorkbook
* @param response
* @param fileName 文件名称
* @throws IOException
*/
public static void downLoadExcelToWebsite(SXSSFWorkbook wb, HttpServletResponse response, String fileName) throws IOException {
response.setHeader("Content-disposition", "attachment; filename="
+ new String((fileName + ".xlsx").getBytes("utf-8"), "ISO8859-1"));//设置下载的文件名
OutputStream outputStream = null;
try {
outputStream = response.getOutputStream();
wb.write(outputStream);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != wb) {
try {
wb.dispose();
} catch (Exception e) {
e.printStackTrace();
}
}
if (null != outputStream) {
try {
outputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
/**
* 导出Excel到本地指定路径
*
* @param totalRowCount 总记录数
* @param titles 标题
* @param exportPath 导出路径
* @param writeExcelDataDelegated 向EXCEL写数据/处理格式的委托类 自行实现
* @throws Exception
*/
public static final void exportExcelToLocalPath(Integer totalRowCount, String[] titles, String exportPath, WriteExcelDataDelegated writeExcelDataDelegated) throws Exception {
logger.info("开始导出：" + formatDate(new Date(), YYYY_MM_DD_HH_MM_SS));
// 初始化EXCEL
SXSSFWorkbook wb = PoiUtil.initExcel(totalRowCount, titles);
// 调用委托类分批写数据
int sheetCount = wb.getNumberOfSheets();
for (int i = 0; i < sheetCount; i++) {
SXSSFSheet eachSheet = wb.getSheetAt(i);
for (int j = 1; j <= PER_SHEET_WRITE_COUNT; j++) {
int currentPage = i * PER_SHEET_WRITE_COUNT + j;
int pageSize = PER_WRITE_ROW_COUNT;
int startRowCount = (j - 1) * PER_WRITE_ROW_COUNT + 1;
int endRowCount = startRowCount + pageSize - 1;
writeExcelDataDelegated.writeExcelData(eachSheet, startRowCount, endRowCount, currentPage, pageSize);
}
}
// 下载EXCEL
PoiUtil.downLoadExcelToLocalPath(wb, exportPath);
logger.info("导出完成：" + formatDate(new Date(),"yyyy-MM-dd HH:mm:ss"));
}
/**
* 将日期转换为字符串
*
* @param date DATE日期
* @param format 转换格式
* @return 字符串日期
*/
public static String formatDate(Date date, String format) {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat(format);
return simpleDateFormat.format(date);
}
public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss";
/**
* 导出Excel到浏览器
*
* @param response
* @param totalRowCount 总记录数
* @param fileName 文件名称
* @param titles 标题
* @param writeExcelDataDelegated 向EXCEL写数据/处理格式的委托类 自行实现
* @throws Exception
*/
public static final void exportExcelToWebsite(HttpServletResponse response, Integer totalRowCount, String fileName, String[] titles, WriteExcelDataDelegated writeExcelDataDelegated) throws Exception {
logger.info("开始导出：" + formatDate(new Date(), YYYY_MM_DD_HH_MM_SS));
long start = System.currentTimeMillis();
// 初始化EXCEL
SXSSFWorkbook wb = PoiUtil.initExcel(totalRowCount, titles);
// 调用委托类分批写数据
int sheetCount = wb.getNumberOfSheets();
ExecutorService exportSheetPool = Executors.newFixedThreadPool(sheetCount);//工头
ArrayList> results = new ArrayList>();
for (int i = 0; i < sheetCount; i++) {
SXSSFSheet eachSheet = wb.getSheetAt(i);
for (int j = 1; j <= PER_SHEET_WRITE_COUNT; j++) {
int currentPage = i * PER_SHEET_WRITE_COUNT + j;
int pageSize = PER_WRITE_ROW_COUNT;
int startRowCount = (j - 1) * PER_WRITE_ROW_COUNT + 1;
int endRowCount = startRowCount + pageSize - 1;
ExportExcelCallable exportExcelCallable = new ExportExcelCallable(writeExcelDataDelegated, eachSheet, startRowCount, endRowCount, currentPage, pageSize);
results.add(exportSheetPool.submit(exportExcelCallable));//submit返回一个Future，代表了即将要返回的结果
}
}
boolean noFinishflag = true;
while(noFinishflag){
for (Future future : results) {
//首先校验是否结束，没有结束直接跳出for循环进行下一次while
if(!future.isDone()){
noFinishflag = true;
break;
}else{
noFinishflag = false;
}
}
}
logger.info("下载前："+formatDate(new Date(), YYYY_MM_DD_HH_MM_SS));
// 下载EXCEL
PoiUtil.downLoadExcelToWebsite(wb, response, fileName);
logger.info("下载后："+formatDate(new Date(), YYYY_MM_DD_HH_MM_SS));
long end = System.currentTimeMillis();
logger.info("导出完成：" + formatDate(new Date(), YYYY_MM_DD_HH_MM_SS)+"总消耗："+(end-start));
}
}
package com.admin.task;
import java.util.concurrent.Callable;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.admin.constant.Consts;
import com.admin.util.WriteExcelDataDelegated;
/**
*
* 导出Excel异步处理类
* 根据多个sheet创建多个线程分别去写不同的sheet，平衡最大处理量和最快处理时间。
*
* @author jeffSheng
* 2019年3月18日
*/
public class ExportExcelCallable implements Callable{
private static final Logger logger = LoggerFactory.getLogger(ExportExcelCallable.class);
private WriteExcelDataDelegated writeExcelDataDelegated;
private SXSSFSheet eachSheet;
private Integer startRowCount;
private Integer endRowCount;
private Integer currentPage;
private Integer pageSize;
public ExportExcelCallable(WriteExcelDataDelegated writeExcelDataDelegated,SXSSFSheet eachSheet,Integer startRowCount,Integer endRowCount,Integer currentPage,Integer pageSize){
this.writeExcelDataDelegated = writeExcelDataDelegated;
this.eachSheet = eachSheet;
this.startRowCount = startRowCount;
this.endRowCount = endRowCount;
this.currentPage=currentPage;
this.pageSize = pageSize;
}
/**
* eachSheet 指定SHEET
startRowCount 开始行
endRowCount 结束行
currentPage 分批查询开始页
pageSize 分批查询数据量
*/
@Override
public String call() throws Exception {
try {
writeExcelDataDelegated.writeExcelData(eachSheet, startRowCount, endRowCount, currentPage, pageSize);
return Consts.Success;
} catch (Exception e) {
e.printStackTrace();
logger.error("导出异常："+e);
}
return Consts.Fail;
}
}
重点说明下一下方法：
/**
* 初始化EXCEL(sheet个数和标题)
*
* @param totalRowCount 总记录数
* @param titles 标题集合
* @return XSSFWorkbook对象
*/
public static SXSSFWorkbook initExcel(Integer totalRowCount, String[] titles) {
logger.info("PER_SHEET_ROW_COUNT：{}，PER_WRITE_ROW_COUNT：{}",PER_SHEET_ROW_COUNT,PER_WRITE_ROW_COUNT);
// 在内存当中保持 100 行 , 超过的数据放到硬盘中在内存当中保持 100 行 , 超过的数据放到硬盘中
SXSSFWorkbook wb = new SXSSFWorkbook(100);
Integer sheetCount = ((totalRowCount % PER_SHEET_ROW_COUNT == 0) ?
(totalRowCount / PER_SHEET_ROW_COUNT) : (totalRowCount / PER_SHEET_ROW_COUNT + 1));
// 根据总记录数创建sheet并分配标题
for (int i = 0; i < sheetCount; i++) {
SXSSFSheet sheet = wb.createSheet("sheet" + (i + 1));
sheet.setRandomAccessWindowSize(-1);
SXSSFRow headRow = sheet.createRow(0);
for (int j = 0; j < titles.length; j++) {
SXSSFCell headRowCell = headRow.createCell(j);
headRowCell.setCellValue(titles[j]);
}
}
return wb;
}
sheet.setRandomAccessWindowSize(-1);//对sheet设置一下无限制访问
这一行如果不添加的话就会报错：
Attempting to write a row[0] in the range [0,0] that is already written to disk.