java读取百万行csv,excel的处理

上篇文章中使用opencsv包读取csv格式,如果csv文件内容过大,会导致jvm产生oom的问题,顾修改代码如下
reader = new CSVReaderBuilder(new InputStreamReader(input, fileEncode)).build()
//记录column
ArrayList columns = new ArrayList<>();
int columnIndex = 0;
String[] rowData = reader.readNext();
复制代码由之前readall 形式改为readnext()一行行读取形式,按行加载至arrayList,按需处理完在对list进行clear()操作,释放内存,再进行readnext加载。。
excel篇:之前采用 poi用户模式读取excel,会极大的消耗内存,查阅资料poi官方推荐使用事件模式读取贴上代码
xls

import org.apache.logging.log4j.LogManager;import org.apache.logging.log4j.Logger;import org.apache.poi.hssf.eventusermodel.*;import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord;import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord;import org.apache.poi.hssf.model.HSSFFormulaParser;import org.apache.poi.hssf.record.*;import org.apache.poi.hssf.usermodel.HSSFDataFormatter;import org.apache.poi.hssf.usermodel.HSSFWorkbook;import org.apache.poi.poifs.filesystem.POIFSFileSystem;import java.io.FileInputStream;import java.util.ArrayList;import java.util.List;/** *  @desc 用于解决.xls2003版本大数据量问题 **/public class ExcelXlsReader implements HSSFListener {    private static final Logger logger = LogManager.getLogger(ExcelXlsxReaderWithDefaultHandler.class);    private int minColums = -1;    private POIFSFileSystem fs;    /**     * 总行数     */    private int totalRows = 0;    /**     * 上一行row的序号     */    private int lastRowNumber;    /**     * 上一单元格的序号     */    private int lastColumnNumber;    /**     * 是否输出formula,还是它对应的值     */    private boolean outputFormulaValues = true;    /**     * 用于转换formulas     */    private EventWorkbookBuilder.SheetRecordCollectingListener workbookBuildingListener;    //excel2003工作簿    private HSSFWorkbook stubWorkbook;    private SSTRecord sstRecord;    private FormatTrackingHSSFListener formatListener;    private final HSSFDataFormatter formatter = new HSSFDataFormatter();    /**     * 文件的绝对路径     */    private String filePath = "";    //表索引    private int sheetIndex = 0;    private BoundSheetRecord[] orderedBSRs;    @SuppressWarnings("unchecked")    private ArrayList boundSheetRecords = new ArrayList();    private int nextRow;    private int nextColumn;    private boolean outputNextStringRecord;    //当前行    private int curRow = 0;    //存储一行记录所有单元格的容器    private ArrayList cellList = new ArrayList();    /**     *     */    private List> cellLists = new ArrayList<>();    /**     *   */    private ArrayList columnList = new ArrayList<>();    /**     * 判断整行是否为空行的标记     */    private boolean flag = false;    @SuppressWarnings("unused")    private String sheetName;    private String companyId = "";    private String containerLogId = "";    private JSONObject config;    private int resultRows = 0;    /**     * 遍历excel下所有的sheet     *     * @param fileName     * @throws Exception     */    public int process(String fileName, String companyId, String containerLogId, JSONObject config) throws Exception {        filePath = fileName;        this.companyId = companyId;        this.containerLogId = containerLogId;        this.config = config;        this.fs = new POIFSFileSystem(new FileInputStream(fileName));        MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this);        formatListener = new FormatTrackingHSSFListener(listener);        HSSFEventFactory factory = new HSSFEventFactory();        HSSFRequest request = new HSSFRequest();        if (outputFormulaValues) {            request.addListenerForAllRecords(formatListener);        } else {            workbookBuildingListener = new EventWorkbookBuilder.SheetRecordCollectingListener(formatListener);            request.addListenerForAllRecords(workbookBuildingListener);        }        factory.processWorkbookEvents(request, fs);        if (cellLists.size() > 0) {  cellLists.clear();        }              return resultRows; //返回该excel文件的总行数,不包括首列和空行    }    /**     * HSSFListener 监听方法,处理Record     * 处理每个单元格     *     * @param record     */    @SuppressWarnings("unchecked")    public void processRecord(Record record) {        try {            int thisRow = -1;            int thisColumn = -1;            String thisStr = null;            String value = null;            switch (record.getSid()) {                case BoundSheetRecord.sid:                    boundSheetRecords.add(record);                    break;                case BOFRecord.sid: //开始处理每个sheet                    BOFRecord br = (BOFRecord) record;                    if (br.getType() == BOFRecord.TYPE_WORKSHEET) {                        //如果有需要,则建立子工作簿                        if (workbookBuildingListener != null && stubWorkbook == null) {                            stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook();                        }                        if (orderedBSRs == null) {                            orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords);                        }                        sheetName = orderedBSRs[sheetIndex].getSheetname();                        sheetIndex++;                    }                    break;                case SSTRecord.sid:                    sstRecord = (SSTRecord) record;                    break;                case BlankRecord.sid: //单元格为空白                    BlankRecord brec = (BlankRecord) record;                    thisRow = brec.getRow();                    thisColumn = brec.getColumn();                    thisStr = "";                    cellList.add(thisColumn, thisStr);                    break;                case BoolErrRecord.sid: //单元格为布尔类型                    BoolErrRecord berec = (BoolErrRecord) record;                    thisRow = berec.getRow();                    thisColumn = berec.getColumn();                    thisStr = berec.getBooleanValue() + "";                    cellList.add(thisColumn, thisStr);                    checkRowIsNull(thisStr);  //如果里面某个单元格含有值,则标识该行不为空行                    break;                case FormulaRecord.sid://单元格为公式类型                    FormulaRecord frec = (FormulaRecord) record;                    thisRow = frec.getRow();                    thisColumn = frec.getColumn();                    if (outputFormulaValues) {                        if (Double.isNaN(frec.getValue())) {                            outputNextStringRecord = true;                            nextRow = frec.getRow();                            nextColumn = frec.getColumn();                        } else {                            thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook,                                    frec.getParsedExpression()) + '"';                        }                    } else {                        thisStr =                                '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"';                    }                    cellList.add(thisColumn, thisStr);                    checkRowIsNull(thisStr);  //如果里面某个单元格含有值,则标识该行不为空行                    break;                case StringRecord.sid: //单元格中公式的字符串                    if (outputNextStringRecord) {                        StringRecord srec = (StringRecord) record;                        thisStr = srec.getString();                        thisRow = nextRow;                        thisColumn = nextColumn;                        outputNextStringRecord = false;                    }                    break;                case LabelRecord.sid:                    LabelRecord lrec = (LabelRecord) record;                    curRow = thisRow = lrec.getRow();                    thisColumn = lrec.getColumn();                    value = lrec.getValue().trim();                    value = value.equals("") ? "" : value;                    cellList.add(thisColumn, value);                    checkRowIsNull(value);  //如果里面某个单元格含有值,则标识该行不为空行                    break;                case LabelSSTRecord.sid: //单元格为字符串类型                    LabelSSTRecord lsrec = (LabelSSTRecord) record;                    curRow = thisRow = lsrec.getRow();                    thisColumn = lsrec.getColumn();                    if (sstRecord == null) {                        cellList.add(thisColumn, "");                    } else {                        value = sstRecord.getString(lsrec.getSSTIndex()).toString().trim();                        value = value.equals("") ? "" : value;                        cellList.add(thisColumn, value);                        checkRowIsNull(value);  //如果里面某个单元格含有值,则标识该行不为空行                    }                    break;                case NumberRecord.sid: //单元格为数字类型                    NumberRecord numrec = (NumberRecord) record;                    curRow = thisRow = numrec.getRow();                    thisColumn = numrec.getColumn();                    //第一种方式                    //value = formatListener.formatNumberDateCell(numrec).trim();//这个被写死,采用的m/d/yy h:mm格式,不符合要求                    //第二种方式,参照formatNumberDateCell里面的实现方法编写                    Double valueDouble = ((NumberRecord) numrec).getValue();                    String formatString = formatListener.getFormatString(numrec);                    if (formatString.contains("m/d/yy") || formatString.contains("yyyy/mm/dd") || formatString.contains(                            "yyyy/m/d")) {                        formatString = "yyyy-MM-dd hh:mm:ss";                    }                    int formatIndex = formatListener.getFormatIndex(numrec);                    value = formatter.formatRawCellContents(valueDouble, formatIndex, formatString).trim();                    value = value.equals("") ? "" : value;                    //向容器加入列值                    cellList.add(thisColumn, value);                    checkRowIsNull(value);  //如果里面某个单元格含有值,则标识该行不为空行                    break;                default:                    break;            }            //遇到新行的操作            if (thisRow != -1 && thisRow != lastRowNumber) {                lastColumnNumber = -1;            }            //空值的操作            if (record instanceof MissingCellDummyRecord) {                MissingCellDummyRecord mc = (MissingCellDummyRecord) record;                curRow = thisRow = mc.getRow();                thisColumn = mc.getColumn();                cellList.add(thisColumn, "");            }            //更新行和列的值            if (thisRow > -1)                lastRowNumber = thisRow;            if (thisColumn > -1)                lastColumnNumber = thisColumn;            //行结束时的操作            if (record instanceof LastCellOfRowDummyRecord) {                if (minColums > 0) {                    //列值重新置空                    if (lastColumnNumber == -1) {                        lastColumnNumber = 0;                    }                }                lastColumnNumber = -1;                if (flag && curRow == 0) {                    columnList = cellList;                }                if (flag && curRow != 0) { //该行不为空行且该行不是第一行,发送(第一行为列名,不需要)                    cellLists.add(cellList);                    totalRows++;                    if (totalRows != 0 && totalRows % 500 == 0) {                       cellLists.clear();                    }                }                //清空容器                cellList = new ArrayList<>();                flag = false;            }        } catch (Exception ex) {            logger.error(ex.getMessage(), ex);        }    }    /**     * 如果里面某个单元格含有值,则标识该行不为空行     *     * @param value     */    public void checkRowIsNull(String value) {        if (value != null && !"".equals(value)) {            flag = true;        }    }}


xlsx

import cn.ifuli.ciq.dataworks.api.Import.flexdata.rpcClient.Client;import com.alibaba.fastjson.JSON;import com.alibaba.fastjson.JSONObject;import org.apache.logging.log4j.LogManager;import org.apache.logging.log4j.Logger;import org.apache.poi.openxml4j.opc.OPCPackage;import org.apache.poi.ss.usermodel.BuiltinFormats;import org.apache.poi.ss.usermodel.DataFormatter;import org.apache.poi.xssf.eventusermodel.XSSFReader;import org.apache.poi.xssf.model.SharedStringsTable;import org.apache.poi.xssf.model.StylesTable;import org.apache.poi.xssf.usermodel.XSSFCellStyle;import org.apache.poi.xssf.usermodel.XSSFRichTextString;import org.xml.sax.Attributes;import org.xml.sax.InputSource;import org.xml.sax.SAXException;import org.xml.sax.XMLReader;import org.xml.sax.helpers.DefaultHandler;import org.xml.sax.helpers.XMLReaderFactory;import java.io.InputStream;import java.math.BigInteger;import java.util.ArrayList;import java.util.List;import java.util.regex.Pattern;/** *@desc POI读取excel有两种模式,一种是用户模式,一种是事件驱动模式 * 采用SAX事件驱动模式解决XLSX文件,可以有效解决用户模式内存溢出的问题, * 该模式是POI官方推荐的读取大数据的模式, * 在用户模式下,数据量较大,Sheet较多,或者是有很多无用的空行的情况下,容易出现内存溢出 * 

* 用于解决.xlsx2007版本大数据量问题 **/public class ExcelXlsxReaderWithDefaultHandler extends DefaultHandler { private static final Logger logger = LogManager.getLogger(ExcelXlsxReaderWithDefaultHandler.class); /** * 单元格中的数据可能的数据类型 */ enum CellDataType { BOOL, ERROR, FORMULA, INLINESTR, SSTINDEX, NUMBER, DATE, NULL } /** * 共享字符串表 */ private SharedStringsTable sst; /** * 上一次的索引值 */ private String lastIndex; /** * 文件的绝对路径 */ private String filePath = ""; /** * 工作表索引 */ private int sheetIndex = 0; /** * sheet名 */ private String sheetName = ""; /** * 总行数 */ private int totalRows = 0; /** * 一行内cell集合 */ private ArrayList cellList = new ArrayList<>(); /** * */ private List> cellLists = new ArrayList<>(); /** * */ private ArrayList columnList = new ArrayList<>(); /** * 判断整行是否为空行的标记 */ private boolean flag = false; /** * 当前行 */ private int curRow = 1; /** * 当前列 */ private int curCol = 0; /** * T元素标识 */ private boolean isTElement; /** * 判断上一单元格是否为文本空单元格 */ private boolean endElementFlag = false; private boolean charactersFlag = false; /** * 异常信息,如果为空则表示没有异常 */ private String exceptionMessage; /** * 单元格数据类型,默认为字符串类型 */ private CellDataType nextDataType = CellDataType.SSTINDEX; private final DataFormatter formatter = new DataFormatter(); /** * 单元格日期格式的索引 */ private short formatIndex; /** * 日期格式字符串 */ private String formatString; //定义前一个元素和当前元素的位置,用来计算其中空的单元格数量,如A6和A8等 private String preRef = null, ref = null; //定义该文档一行最大的单元格数,用来补全一行最后可能缺失的单元格 private String maxRef = null; /** * 单元格 */ private StylesTable stylesTable; private String companyId = ""; private String containerLogId = ""; private JSONObject config; private int resultRows = 0; //private int testResult = 0; /** * 遍历工作簿中所有的电子表格 * 并缓存在mySheetList中 * * @param filename * @throws Exception */ public int process(String filename, String companyId, String containerLogId, JSONObject config) throws Exception { filePath = filename; this.companyId = companyId; this.containerLogId = containerLogId; this.config = config; OPCPackage pkg = OPCPackage.open(filename); XSSFReader xssfReader = new XSSFReader(pkg); stylesTable = xssfReader.getStylesTable(); SharedStringsTable sst = xssfReader.getSharedStringsTable(); XMLReader parser = XMLReaderFactory.createXMLReader("org.apache.xerces.parsers.SAXParser"); this.sst = sst; parser.setContentHandler(this); XSSFReader.SheetIterator sheets = (XSSFReader.SheetIterator) xssfReader.getSheetsData();// while (sheets.hasNext()) { //遍历sheet curRow = 1; //标记初始行为第一行 sheetIndex++; InputStream sheet = sheets.next(); //sheets.next()和sheets.getSheetName()不能换位置,否则sheetName报错 sheetName = sheets.getSheetName(); InputSource sheetSource = new InputSource(sheet); parser.parse(sheetSource); //解析excel的每条记录,在这个过程中startElement()、characters()、endElement()这三个函数会依次执行 sheet.close();// } if (cellLists.size() > 0) { cellLists.clear(); } //组装数据并发送 UserData data = new UserData(); data.setCompanyId(companyId); data.setContainerLogId(containerLogId); data.setUserDataList(new ArrayList<>()); data.setIsEnd(true); return resultRows; //返回该excel文件的总行数,不包括首列和空行 } /** * 第一个执行 * * @param uri * @param localName * @param name * @param attributes * @throws SAXException */ @Override public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException { //c => 单元格 if ("c".equals(name)) { //前一个单元格的位置 if (preRef == null) { preRef = attributes.getValue("r"); } else { //判断前一次是否为文本空字符串,true则表明不是文本空字符串,false表明是文本空字符串跳过把空字符串的位置赋予preRef if (endElementFlag) { preRef = ref; } } //当前单元格的位置 ref = attributes.getValue("r"); //设定单元格类型 this.setNextDataType(attributes); endElementFlag = false; charactersFlag = false; } //当元素为t时 if ("t".equals(name)) { isTElement = true; } else { isTElement = false; } //置空 lastIndex = ""; } /** * 第二个执行 * 得到单元格对应的索引值或是内容值 * 如果单元格类型是字符串、INLINESTR、数字、日期,lastIndex则是索引值 * 如果单元格类型是布尔值、错误、公式,lastIndex则是内容值 * * @param ch * @param start * @param length * @throws SAXException */ @Override public void characters(char[] ch, int start, int length) throws SAXException { charactersFlag = true; lastIndex += new String(ch, start, length); } /** * 第三个执行 * * @param uri * @param localName * @param name * @throws SAXException */ @Override public void endElement(String uri, String localName, String name) throws SAXException { try { //t元素也包含字符串 if (isTElement) {//这个程序没经过 //将单元格内容加入rowlist中,在这之前先去掉字符串前后的空白符 String value = lastIndex.trim(); cellList.add(curCol, value); endElementFlag = true; curCol++; isTElement = false; //如果里面某个单元格含有值,则标识该行不为空行 if (value != null && !"".equals(value)) { flag = true; } } else if ("v".equals(name)) { //v => 单元格的值,如果单元格是字符串,则v标签的值为该字符串在SST中的索引 String value = this.getDataValue(lastIndex.trim(), "");//根据索引值获取对应的单元格值 //补全单元格之间的空单元格 if (!ref.equals(preRef)) { int len = countNullCell(ref, preRef); for (int i = 0; i < len; i++) { cellList.add(curCol, ""); curCol++; } } cellList.add(curCol, value); curCol++; endElementFlag = true; //如果里面某个单元格含有值,则标识该行不为空行 if (value != null && !"".equals(value)) { flag = true; } } else { //如果标签名称为row,这说明已到行尾,调用optRows()方法 if ("row".equals(name)) { //默认第一行为表头,以该行单元格数目为最大数目 if (curRow == 1) { maxRef = ref; } //补全一行尾部可能缺失的单元格 if (maxRef != null) { int len = -1; //前一单元格,true则不是文本空字符串,false则是文本空字符串 if (charactersFlag) { len = countNullCell(maxRef, ref); } else { len = countNullCell(maxRef, preRef); } for (int i = 0; i <= len; i++) { cellList.add(curCol, ""); curCol++; } } if (flag && curRow == 1) { columnList = (cellList); } if (flag && curRow != 1) { //该行不为空行且该行不是第一行,则发送(第一行为列名,不需要) cellLists.add(cellList); totalRows++; if (totalRows != 0 && totalRows % 500 == 0) { cellLists.clear(); } //ExcelReaderUtil.sendRows(filePath, sheetName, sheetIndex, curRow, cellList); } cellList = new ArrayList<>(); curRow++; curCol = 0; preRef = null; ref = null; flag = false; } } } catch (Exception ex) { logger.error(ex.getMessage(), ex); } } /** * 处理数据类型 * * @param attributes */ public void setNextDataType(Attributes attributes) { nextDataType = CellDataType.NUMBER; //cellType为空,则表示该单元格类型为数字 formatIndex = -1; formatString = null; String cellType = attributes.getValue("t"); //单元格类型 String cellStyleStr = attributes.getValue("s"); // String columnData = attributes.getValue("r"); //获取单元格的位置,如A1,B1 if ("b".equals(cellType)) { //处理布尔值 nextDataType = CellDataType.BOOL; } else if ("e".equals(cellType)) { //处理错误 nextDataType = CellDataType.ERROR; } else if ("inlineStr".equals(cellType)) { nextDataType = CellDataType.INLINESTR; } else if ("s".equals(cellType)) { //处理字符串 nextDataType = CellDataType.SSTINDEX; } else if ("str".equals(cellType)) { nextDataType = CellDataType.FORMULA; } if (cellStyleStr != null) { //处理日期 int styleIndex = Integer.parseInt(cellStyleStr); XSSFCellStyle style = stylesTable.getStyleAt(styleIndex); formatIndex = style.getDataFormat(); formatString = style.getDataFormatString(); if (formatString.contains("m/d/yyyy") || formatString.contains("yyyy/mm/dd") || formatString.contains( "yyyy/m/d")||formatString.contains("m/d/yy")) { nextDataType = CellDataType.DATE; formatString = "yyyy-MM-dd hh:mm:ss"; } if (formatString == null) { nextDataType = CellDataType.NULL; formatString = BuiltinFormats.getBuiltinFormat(formatIndex); } } } /** * 对解析出来的数据进行类型处理 * * @param value 单元格的值, * value代表解析:BOOL的为0或1, ERROR的为内容值,FORMULA的为内容值,INLINESTR的为索引值需转换为内容值, * SSTINDEX的为索引值需转换为内容值, NUMBER为内容值,DATE为内容值 * @param thisStr 一个空字符串 * @return */ @SuppressWarnings("deprecation") public String getDataValue(String value, String thisStr) { switch (nextDataType) { // 这几个的顺序不能随便交换,交换了很可能会导致数据错误 case BOOL: //布尔值 char first = value.charAt(0); thisStr = first == '0' ? "FALSE" : "TRUE"; break; case ERROR: //错误 thisStr = "\"ERROR:" + value.toString() + '"'; break; case FORMULA: //公式 thisStr = '"' + value.toString() + '"'; break; case INLINESTR: XSSFRichTextString rtsi = new XSSFRichTextString(value.toString()); thisStr = rtsi.toString(); rtsi = null; break; case SSTINDEX: //字符串 String sstIndex = value.toString(); try { int idx = Integer.parseInt(sstIndex); XSSFRichTextString rtss = new XSSFRichTextString(sst.getEntryAt(idx));//根据idx索引值获取内容值 thisStr = rtss.toString(); //System.out.println(thisStr); //有些字符串是文本格式的,但内容却是日期 rtss = null; } catch (NumberFormatException ex) { thisStr = value.toString(); } break; case NUMBER: //数字 if (formatString != null) { if(formatString.contains("reserved")){ thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, "yyyy-MM-dd hh:mm:ss"); }else{ thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, formatString).trim(); } } else { thisStr = getRealStringValueOfDouble(Double.parseDouble(value)); } thisStr = thisStr.replace("_", "").trim(); break; case DATE: //日期 thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, formatString); // 对日期字符串作特殊处理,去掉T thisStr = thisStr.replace("T", " "); break; default: thisStr = " "; break; } return thisStr; } public int countNullCell(String ref, String preRef) { //excel2007最大行数是1048576,最大列数是16384,最后一列列名是XFD String xfd = ref.replaceAll("\\d+", ""); String xfd_1 = preRef.replaceAll("\\d+", ""); xfd = fillChar(xfd, 3, '@', true); xfd_1 = fillChar(xfd_1, 3, '@', true); char[] letter = xfd.toCharArray(); char[] letter_1 = xfd_1.toCharArray(); int res = (letter[0] - letter_1[0]) * 26 * 26 + (letter[1] - letter_1[1]) * 26 + (letter[2] - letter_1[2]); return res - 1; } public String fillChar(String str, int len, char let, boolean isPre) { int len_1 = str.length(); if (len_1 < len) { if (isPre) { for (int i = 0; i < (len - len_1); i++) { str = let + str; } } else { for (int i = 0; i < (len - len_1); i++) { str = str + let; } } } return str; } /** * @return the exceptionMessage */ public String getExceptionMessage() { return exceptionMessage; } private static String getRealStringValueOfDouble(Double d) { String doubleStr = d.toString(); boolean b = doubleStr.contains("E"); int indexOfPoint = doubleStr.indexOf('.'); if (b) { int indexOfE = doubleStr.indexOf('E'); BigInteger xs = new BigInteger(doubleStr.substring(indexOfPoint + BigInteger.ONE.intValue(), indexOfE)); int pow = Integer.valueOf(doubleStr.substring(indexOfE + BigInteger.ONE.intValue())); int xsLen = xs.toByteArray().length; int scale = xsLen - pow > 0 ? xsLen - pow : 0; doubleStr = String.format("%." + scale + "f", d); } else { java.util.regex.Pattern p = Pattern.compile(".0$"); java.util.regex.Matcher m = p.matcher(doubleStr); if (m.find()) { doubleStr = doubleStr.replace(".0", ""); } } return doubleStr; }}

以上代码需引用common.io架包

最后推荐监控jvm内存cpu线程工具

visual vm

java读取百万行csv,excel的处理_第1张图片

你可能感兴趣的:(java读取百万行csv,excel的处理)