java springmvc用线程池高效导出百万级csv数据,前端展现进度条
领导说公司的报表导出太慢还没有进度条展示,以前用的poi操作excel导出xls,从数据库查询数据,写入本地文件,然后读取本地文件下载效率低下,特别是数据量大的时候。所以我对后台cms系统导出做了优化，采用导出csv格式，这种格式下速度比xls快很多。
**csv和xls的区别**
经常会用户报表导出格式为csv和xls格式。他们的性能相差很大，我认为csv是非常高效的格式，而xls性能虽比csv差很多，但xls可以把格式做得很完美。现在我将两种不同格式做一些整理，希望对你们有点用。
xls 文件就是Microsoft excel电子表格的文件格式。我想就不用多介绍了吧，学校里多少都学过的。
CSV是最通用的一种文件格式，它可以非常容易地被导入各种PC表格及数据库中。 此文件，一行即为数据表的一行。生成数据表字段用半角逗号隔开。
CSV是文本文件,用记事本就能打开,XLS是二进制的文件只有用EXCEL才能打
CSV（以逗号分隔）
CSV (*.csv) 文件格式只能保存活动工作表中的单元格所显示的文本和数值。工作表中所有的数据行和字符都将保存。数据列以逗号分隔，每一行数据都以回车符结束。如果单元格中包含逗号，则该单元格中的内容以双引号引起。
如果单元格显示的是公式而不是数值，该公式将转换为文本方式。所有格式、图形、对象和工作表的其他内容将全部丢失。欧元符号将转换为问号。
//工具类
import java.io.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.sf.utils.core.StringUtil;
import com.sf.tuxiaoer.service.SfSQLBaseService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
//csv导出工具类
/**
-
功能描述 CSV下载工具类 实现Callable接口call()方法
-
@author hjli
-
@date 2019/7/13
-
@param
-
@return
*/
public class CsvDownload implements Callable {
private static final Logger logger = LoggerFactory.getLogger(XlsDownload.class);
private int index;
private int defaultNums;
private String sql_select;
private String[] columns;
private SfSQLBaseService sfSQLBaseService;
private CountDownLatch latch;
private HttpServletRequest request;
}public CsvDownload (){
}
/**
- 功能描述
- @author hjli
- @date 2019/7/13
- @param index,//数据查询分页下标
- defaultNums, //默认一页的数量
- sql_select, //sql查询语句
- columns, //列名
- sfSQLBaseService, //jdbctemplate
- latch, //CountDownLatch计时器
- request //HttpServletRequest 请求
- @return
*/
public CsvDownload(int index,int defaultNums,String sql_select,
String[] columns,SfSQLBaseService sfSQLBaseService,
CountDownLatch latch,HttpServletRequest request){
this.index = index;
this.defaultNums = defaultNums;
this.sql_select = sql_select;
this.columns = columns;
this.sfSQLBaseService = sfSQLBaseService;
this.latch = latch;
this.request = request;
}
//返回csv后缀
private String getFileName(int rowSize) {
String dateString = DateTimeUtils.getServerTime(“yyyyMMddHHmmss”);
String fileName = dateString + “_” + rowSize + “.csv”;
return fileName;
}
/**
* 功能描述 CSV下载
* @author hjli
* @date 2019/7/13
* @param request,
* response, //响应
* sfSQLBaseService, //jdbctemplate
* title,//表头名
* column, //列名
* sql_select, //sql查询语句
* sql_count //查询总数
* @return void
*/
public void csvDownLoad(HttpServletRequest request,
HttpServletResponse response,
SfSQLBaseService sfSQLBaseService,
String title, String column, String sql_select, String sql_count) throws InterruptedException {
String start_date = "";
String end_date = "";
//记录开始时间
start_date = DateTimeUtils.getServerTime();
//用于存储数据
StringBuffer stringBuffer = new StringBuffer(1000);
String[] titles = StringUtil.split(title,",");
String[] columns = StringUtil.split(column,",");
//告诉前端下载选择类型 导出类别 1:多线程、 2单线程
request.getSession().setAttribute("downloadType", 1);
//得到数据总量
int rowSize = getSize(sfSQLBaseService, sql_count);
//根据数据总量分配线程
int defaultNums = 0;
if(rowSize <= 3000){
defaultNums = 3000;
}else if(rowSize > 3000 && rowSize <= 10000){
defaultNums = 5000;
}else if(rowSize > 10000 && rowSize <= 50000){
defaultNums = 10000;
}else if(rowSize > 50000 && rowSize <= 100000){
defaultNums = 50000;
}else{
defaultNums = 100000;
}
//插入表头内容
for(int x = 0; x< titles.length; x++){
stringBuffer.append(titles[x]);
if(x < titles.length - 1){
stringBuffer.append(",");
}
}
//换行
stringBuffer.append("\r\n");
//分成几页
int sheetNums = (int) getSheetNums(rowSize, defaultNums);
//数据存储起来用于前端展示进度条
//数据总量
request.getSession().setAttribute("total_data",rowSize);
//数据包总量
request.getSession().setAttribute("total_packet",sheetNums);
//手动创建线程池 根据页数动态创建线程池
ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("thread-call-runner-%d").build();
ExecutorService es = new ThreadPoolExecutor(sheetNums,sheetNums,0L,TimeUnit.MILLISECONDS,new LinkedBlockingQueue(),namedThreadFactory);
//动态创建计数器
CountDownLatch latch = new CountDownLatch(sheetNums);
//存储Future数据
List resultList = new ArrayList<>();
try {
for(int i=0;i future = es.submit
(new XlsDownload(i, defaultNums, sql_select, columns, sfSQLBaseService,latch,request));
resultList.add(future);
}
//阻塞主进程,等待latch.getCount()为0时放行
latch.await();
}catch (Exception e){
e.printStackTrace();
}finally {
es.shutdownNow();//关闭线程池
//释放session资源
request.getSession().setAttribute("progress_flag",false);
request.getSession().removeAttribute("total_data");
request.getSession().removeAttribute("total_packet");
for(int i=0;i>end:"+end_date);
try {
//页面响应
out = response.getOutputStream();
out.write(stringBuffer.toString().getBytes("UTF-8"));
}catch (Exception e){
e.printStackTrace();
}finally {
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Override
public String call() throws Exception {
String tmpSelect = sql_select + " LIMIT " + index * defaultNums + ", " + defaultNums;
List dataList = sfSQLBaseService.queryForList(tmpSelect);
String _status = "";
StringBuffer sb = new StringBuffer();
for(int i=0;i
";
request.getSession().setAttribute("thread_"+index,_status);
}
sb.append("\r\n");
request.getSession().setAttribute("size_"+index,(i+1));
request.getSession().setAttribute("count_"+index,index);
request.getSession().setAttribute("packet_"+index,index);
}
logger.info("线程:{}执行完毕,线程ID：{}",index,Thread.currentThread().getName());
latch.countDown();//线程每次执行完毕countDown
return sb.toString();
}
前面用session存储了很多数据,主要是用于前端页面进度条展示,这里需要创建一个进度条监听的controller
//controller
import com.sf.tuxiaoer.web.BaseController;
import net.sf.json.JSONArray;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import javax.servlet.http.HttpServletResponse;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Map;
/**
* 功能描述 进度条监听
* @author hjli
* @date 2019/7/13
* @param
* @return
*/
@Controller
public class ProgressBar extends BaseController {
private DecimalFormat df = new DecimalFormat("0.00");
/**
* 功能描述 进度条监听
* @author hjli
* @date 2019/7/13
* @param response
* @return void
*/
@RequestMapping("/wash/progress/progressBarStatu")
public void progressBarStatu(HttpServletResponse response){
Map dataMap = new HashMap();
try {
boolean progress_flag = (boolean)request.getSession().getAttribute("progress_flag");
//默认不展示
boolean showFlag = false;
//在导数据
if(progress_flag == true){
//导出类别 1:多线程、 2单线程
int downloadType = request.getSession().getAttribute("downloadType") != null ? (int)request.getSession().getAttribute("downloadType") : 9;
//多线程
if(downloadType == 1){
int total_data =request.getSession().getAttribute("total_data") != null ? (int)request.getSession().getAttribute("total_data") : 0;
int total_packet = request.getSession().getAttribute("total_packet") != null ? (int)request.getSession().getAttribute("total_packet") : 0;
//数据总量
dataMap.put("total_packet",total_packet);
//数据包
dataMap.put("total_data",total_data);
//已经有线程在下载时
if(total_packet > 0){
int _count = 0;
int _size = 0;
StringBuffer stringBuffer = new StringBuffer();
for(int i=0;i 0){
//展现下载状态
showFlag = true;
//实时下载进度
String status = (String)request.getSession().getAttribute("status");
dataMap.put("status",status);
dataMap.put("total_size",total_size);
//下载百分比
float progress = (float)size * 100 / total_size;
dataMap.put("progress",df.format(progress)+"%");
}
}
//导出类别 1:多线程、 2单线程
dataMap.put("downloadType",downloadType);
}
//是否展现下载状态 true展现 false不展现
dataMap.put("showFlag",showFlag);
//进度条状态 true显示 false隐藏
dataMap.put("progress_flag",progress_flag);
PrintWriter out = response.getWriter();
out.print(JSONArray.fromObject(dataMap).toString());
out.close();
}catch (Exception e){e.printStackTrace();}
}
//前端js页面展示、可写在公共jsp页面这样所有页面都能调用进度条
//CSS
.mydiv {
-moz-border-radius: 16px;
-webkit-border-radius: 16px;
border-radius: 16px;
background-color:lavender;
text-align: center;
z-index:99;
width:38%;
height: auto;
left:38%;/*FF IE7*/
top: 8%;/*FF IE7*/
margin-left:-130px!important;/*FF IE7 该值为本身宽的一半 */
margin-top:-20px!important;/*FF IE7 该值为本身高的一半*/
margin-top:0px;
position:fixed!important;/*FF IE7*/
position:absolute;/*IE6*/
}
.bbg {
background-color: #878b80;
width: 100%;
height: 100%;
left:0;
top:0;/*FF IE7*/
filter:alpha(opacity=50);/*IE*/
opacity:0.5;/*FF*/
z-index:1;
position:fixed!important;/*FF IE7*/
position:absolute;/*IE6*/
}
//html
//js
var interval_time = 1000;
var time = 0;
//一直唤醒
var task = window.setInterval(function(){
time++;
$.ajax({
url:'<%=basePath %>wash/progress/progressBarStatu',
type:'post',
success:function(data){
var objs = jQuery.parseJSON(data);
if(objs[0].progress_flag == true){
$("#bbg").css({display:'block'});
$("#bpopDiv").css({display:'block'});
var msg = "数据下载中,请稍后......
" +
"耗时:"+time+"s
" ;
if(objs[0].showFlag == true){
//导出类别 1:多线程、 2单线程
if(objs[0].downloadType == 1){
msg += "总资源量:"+objs[0].total_data+"
" +
"数据包:"+objs[0]._count+" / "+objs[0].total_packet+"
" ;
msg += ""+objs[0].stringBuffer+"
" +
"总进度:"+objs[0].progress+"";
}else if(objs[0].downloadType == 2){
msg += ""+objs[0].status+"
" +
"下载进度:"+objs[0].progress+"" ;
}
$("#progress_bar").html(msg);
}
}else{
time = 0;
//window.clearInterval(task);
$("#progress_bar").html("数据准备中,请稍等...");
$("#bbg").css({display:'none'});
$("#bpopDiv").css({display:'none'});
}
}
});
},interval_time);