csv导出百万级数据
1：测试数据200多万，项目使用
2：csv工具类，我用的第一方法
package com.wptx.comm.config;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang3.StringUtils;
public class CSVUtils {
private static CSVUtils csvUtils;
public static CSVUtils getInstance() {
if (csvUtils == null) {
csvUtils = new CSVUtils();
}
return csvUtils;
}
/**
* 生成为CVS文件
* @param exportData
* 源数据List
* @param map
* csv文件的列表头map
* @param outPutPath
* 文件路径
* @param fileName
* 文件名称
* @return
*/
@SuppressWarnings({ "rawtypes","unused" })
public static String createCSVFile(List exportData, LinkedHashMap map, String outPutPath, String fileName) {
File csvFile = null;
BufferedWriter csvFileOutputStream = null;
String path = null;
try {
File file = new File(outPutPath);
if (!file.exists()) {
file.mkdirs();
}
// 定义文件名格式并创建
csvFile = File.createTempFile(fileName, ".csv", new File(outPutPath));
path = csvFile.getCanonicalPath();
// System.out.println(csvFile.getName());
// System.out.println("csvFile：" + path);
// UTF-8使正确读取分隔符"," osw.write(new String(new byte[] { (byte) 0xEF, (byte) 0xBB,(byte) 0xBF }));
// 如果生产文件乱码，windows下用gbk，linux用UTF-8
csvFileOutputStream = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), "UTF-8"), 1024);
csvFileOutputStream.write('\ufeff');
// 写入文件头部
for (Iterator propertyIterator = map.entrySet().iterator(); propertyIterator.hasNext();) {
java.util.Map.Entry propertyEntry = (java.util.Map.Entry) propertyIterator.next();
csvFileOutputStream.write((String) propertyEntry.getValue() != null ? (String) propertyEntry.getValue() : "");
if (propertyIterator.hasNext()) {
csvFileOutputStream.write(",");
}
}
csvFileOutputStream.newLine();
// 写入文件内容
for (Iterator iterator = exportData.iterator(); iterator.hasNext();) {
Object row = (Object) iterator.next();
for (Iterator propertyIterator = map.entrySet().iterator(); propertyIterator.hasNext();) {
java.util.Map.Entry propertyEntry = (java.util.Map.Entry) propertyIterator.next();
// csvFileOutputStream.write((String) BeanUtils.getProperty(row, (String) propertyEntry.getValue() != null ? (String) propertyEntry.getValue() : ""));
csvFileOutputStream.write((String) BeanUtils.getProperty(row, (String) propertyEntry.getKey()));
if (propertyIterator.hasNext()) {
csvFileOutputStream.write(",");
}
}
if (iterator.hasNext()) {
csvFileOutputStream.newLine();
}
}
csvFileOutputStream.flush();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
csvFileOutputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return csvFile.getName();
}
// /**
// * 生成为CVS文件
// * @param exportData
// * 源数据List
// * @param map
// * csv文件的列表头map
// * @param outPutPath
// * 文件路径
// * @param fileName
// * 文件名称
// * @return
// */
// @SuppressWarnings({ "rawtypes","unused" })
// public static String createCSVFile(List exportData, LinkedHashMap map, String outPutPath, String fileName) {
// File csvFile = null;
// BufferedWriter csvFileOutputStream = null;
// String path = null;
// try {
// File file = new File(outPutPath);
// if (!file.exists()) {
// file.mkdir();
// }
// // 定义文件名格式并创建
// csvFile = File.createTempFile(fileName, ".csv", new File(outPutPath));
// path = csvFile.getCanonicalPath();
System.out.println(csvFile.getName());
System.out.println("csvFile：" + path);
// // UTF-8使正确读取分隔符"," osw.write(new String(new byte[] { (byte) 0xEF, (byte) 0xBB,(byte) 0xBF }));
// // 如果生产文件乱码，windows下用gbk，linux用UTF-8
// // csvFileOutputStream = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), "GBK"), 1024);
// CsvWriter csvWriter = new CsvWriter(csvFile.getAbsolutePath(), ',', Charset.forName("UTF-8"));
// // csvWriter.write(new String(new byte[] { (byte) 0xEF, (byte) 0xBB,(byte) 0xBF }));
// String chatset = new String(new byte[] { (byte) 0xEF, (byte) 0xBB,(byte) 0xBF });
//
// String[] titleArray = new String[map.size()];
// // 写入文件头部
// int i = 0;
// for (Iterator propertyIterator = map.entrySet().iterator(); propertyIterator.hasNext();) {
// java.util.Map.Entry propertyEntry = (java.util.Map.Entry) propertyIterator.next();
// // csvFileOutputStream.write('\ufeff');
// // csvFileOutputStream.write((String) propertyEntry.getValue() != null ? (String) propertyEntry.getValue() : "");
// // csvWriter.write((String) propertyEntry.getValue() != null ? (String) propertyEntry.getValue() : "");
// if (i == 0) {
// titleArray[i++] = (String) propertyEntry.getValue() != null ? chatset + (String) propertyEntry.getValue() : "";
// }
// else {
// titleArray[i++] = (String) propertyEntry.getValue() != null ? (String) propertyEntry.getValue() : "";
// }
//
if (propertyIterator.hasNext()) {
// csvFileOutputStream.write(",");
// csvWriter.write(",");
}
// }
//
// csvWriter.writeRecord(titleArray, true);
//
//
// // csvFileOutputStream.newLine();
// // csvWriter.write("\r\n");
//
// // 写入文件内容
// for (Iterator iterator = exportData.iterator(); iterator.hasNext();) {
// Object row = (Object) iterator.next();
// String[] datdaArray = new String[map.size()];
// i = 0;
// for (Iterator propertyIterator = map.entrySet().iterator(); propertyIterator.hasNext();) {
// java.util.Map.Entry propertyEntry = (java.util.Map.Entry) propertyIterator.next();
// // csvFileOutputStream.write('\ufeff');
// // csvFileOutputStream.write((String) BeanUtils.getProperty(row, (String) propertyEntry.getKey()));
// // csvWriter.write((String) BeanUtils.getProperty(row, (String) propertyEntry.getKey()));
// datdaArray[i++] = (String) BeanUtils.getProperty(row, (String) propertyEntry.getKey());
//
if (propertyIterator.hasNext()) {
// csvFileOutputStream.write(",");
// csvWriter.write(",");
}
// }
//
// csvWriter.writeRecord(datdaArray, true);
//
if (iterator.hasNext()) {
// csvFileOutputStream.newLine();
// csvWriter.write("\r\n");
}
// }
//
//
// // csvFileOutputStream.flush();
// csvWriter.close();
// } catch (Exception e) {
// e.printStackTrace();
// } finally {
try {
csvFileOutputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
// }
// return csvFile.getName();
// }
/**
* 下载文件
* @param response
* @param csvFilePath
* 文件路径
* @param fileName
* 文件名称
* @throws IOException
*
*/
public static void exportFile(HttpServletResponse response, String csvFilePath, String fileName) throws IOException {
response.setContentType("application/csv;charset=UTF-8");
response.setHeader("Content-Disposition", "attachment; filename=" + URLEncoder.encode(fileName, "UTF-8"));
InputStream in = null;
try {
in = new FileInputStream(csvFilePath);
int len = 0;
byte[] buffer = new byte[1024];
response.setCharacterEncoding("UTF-8");
OutputStream out = response.getOutputStream();
while ((len = in.read(buffer)) > 0) {
out.write(new byte[] { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF });
out.write(buffer, 0, len);
}
} catch (FileNotFoundException e) {
//System.out.println(e);
} finally {
if (in != null) {
in.close();
}
}
}
/**
* 生成csv文件之前特殊字符转义
* @param str
* @return
*/
public static String csvString(String str){
str=str.toString().replaceAll("[/r/n]"," ");
if(str.contains("-")){
str = str.replace("\"","\"\"");
str ="\"" + str + "\"";
}
return str;
}
/**
* 测试数据
* @param args
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] args) {
List exportData = new ArrayList3：业务层。我导出了19个字段。exportData：经过转化后的List集合。titleMap：表头。这两个数据的数字是相互对应了，剩余的两个参数是，导出的路径和文件名
//查询需要导出的数据
List dataList = mapper.export(taskId);
List> exportData = new ArrayList>();
for(TaskStoreItemDTO tsd:dataList) {
LinkedHashMap row = new LinkedHashMap();
if(!_isNullOrEmpty(tsd.getTabNbr())) {
row.put("1", tsd.getTabNbr());
}else {
row.put("1", "");
}
if(!_isNullOrEmpty(tsd.getRegionName())) {
row.put("2", tsd.getRegionName());
}else {
row.put("2", "");
}
if(!_isNullOrEmpty(tsd.getSmallRegionName())) {
row.put("3", tsd.getSmallRegionName());
}else {
row.put("3", "");
}
if(!_isNullOrEmpty(tsd.getClubId())) {
row.put("4", tsd.getClubId());
}else {
row.put("4", "");
}
if(!_isNullOrEmpty(tsd.getDeptNbr())) {
row.put("5", tsd.getDeptNbr());
}else {
row.put("5", "");
}
if(!_isNullOrEmpty(tsd.getItemNbr())) {
row.put("6", tsd.getItemNbr());
}else {
row.put("6", "");
}
if(!_isNullOrEmpty(tsd.getSerialNbr())) {
row.put("7", tsd.getSerialNbr());
}else {
row.put("7", "");
}
if(!_isNullOrEmpty(tsd.getUpcNbr())) {
row.put("8", tsd.getUpcNbr());
}else {
row.put("8", "");
}
if(!_isNullOrEmpty(tsd.getGoodsName())) {
row.put("9", tsd.getGoodsName());
}else {
row.put("9", "");
}
if(!_isNullOrEmpty(tsd.getSalesType())) {
row.put("10", tsd.getSalesType());
}else {
row.put("10", "");
}
if(!_isNullOrEmpty(tsd.getFixtureName())) {
row.put("11", tsd.getFixtureName());
}else {
row.put("11", "");
}
if(!_isNullOrEmpty(tsd.getOperatorTime())) {
row.put("12", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tsd.getOperatorTime()));
}else {
row.put("12","");
}
if(!_isNullOrEmpty(tsd.getHaveFeedbackStatus())) {
if(tsd.getHaveFeedbackStatus()==0) {
row.put("13", "否");
}else {
row.put("13", "是");
}
}else {
row.put("13", "");
}
if(!_isNullOrEmpty(tsd.getFeedbackStatus())) {
if(tsd.getFeedbackStatus()==0) {
row.put("14", "未拍照");
}else {
row.put("14", "已拍照");
}
}else {
row.put("14", "");
}
if(!_isNullOrEmpty(tsd.getOperateReasonContent())) {
row.put("15", tsd.getOperateReasonContent());
}else {
row.put("15","");
}
if(!_isNullOrEmpty(tsd.getDisplayStatus())) {
if(tsd.getDisplayStatus()==0) {
row.put("16", "待分配");
}
if(tsd.getDisplayStatus()==1) {
row.put("16", "未陈列");
}
if(tsd.getDisplayStatus()==2) {
row.put("16", "已陈列");
}
if(tsd.getDisplayStatus()==3) {
row.put("16", "无法陈列");
}
}else {
row.put("16", "");
}
if(!_isNullOrEmpty(tsd.getAuditStatus())) {
if(tsd.getAuditStatus()==0) {
row.put("17", "未提交审核");
}
if(tsd.getAuditStatus()==1) {
row.put("17", "待审核");
}
if(tsd.getAuditStatus()==2) {
row.put("17", "已审核");
}
}else {
row.put("17", "");
}
if(!_isNullOrEmpty(tsd.getSubmitHoro())) {
if(tsd.getSubmitHoro()==0) {
row.put("18", "否");
}
if(tsd.getSubmitHoro()==1) {
row.put("18", "是");
}
}else {
row.put("18", "");
}
if(!_isNullOrEmpty(tsd.getRemark())) {
row.put("19", tsd.getRemark());
}else {
row.put("19", "");
}
exportData.add(row);
}
LinkedHashMap titleMap = new LinkedHashMap();
titleMap.put("1", "档期");
titleMap.put("2", "大区");
titleMap.put("3", "小区");
titleMap.put("4", "门店号");
titleMap.put("5", "部门号");
titleMap.put("6", "itemNbr");
titleMap.put("7", "陈列序号");
titleMap.put("8", "UPC");
titleMap.put("9", "商品名称");
titleMap.put("10", "促销类型");
titleMap.put("11", "陈列类型");
titleMap.put("12", "陈列时间");
titleMap.put("13", "是否需要拍照");
titleMap.put("14", "是否已拍照");
titleMap.put("15", "无法执行原因");
titleMap.put("16", "商品状态");
titleMap.put("17", "审核状态");
titleMap.put("18", "是否提交总部");
titleMap.put("19", "备注");
//导出的路径
String filePath=serviceConfig.getExportFeedbackPath();
//导出的文件名称
String filename="商品陈列清单";
String newFilePath = CSVUtils.getInstance().createCSVFile(exportData, titleMap, filePath, filename);