解决使用poi处理execl表格内存溢出问题

在平常的开发中会用到处理表格文件的功能，poi就是一个非常优秀的处理表格的java框架，但是当表格文件的数据量过大处理过程就会出现堆内存溢出的异常，让人痛苦了两天，最后在谷歌的帮助下找了一个解决方法，现在把这个方法分享出来希望可以帮助到大家！！

import org.apache.poi.hssf.eventusermodel.*;
import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord;
import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord;
import org.apache.poi.hssf.model.HSSFFormulaParser;
import org.apache.poi.hssf.record.*;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;

public class XlsToCsv implements HSSFListener {

    private int minColumns;
    private POIFSFileSystem fs;
    private PrintStream output;

    private int lastRowNumber;
    private int lastColumnNumber;
    public long time_cha;

    public long getTime_cha() {
        return time_cha;
    }

    public void setTime_cha(long time_cha) {
        this.time_cha = time_cha;
    }

    /**
     * Should we output the formula, or the value it has?
     */
    private boolean outputFormulaValues = true;

    /**
     * For parsing Formulas
     */
    private EventWorkbookBuilder.SheetRecordCollectingListener workbookBuildingListener;
    private HSSFWorkbook stubWorkbook;

    // Records we pick up as we process
    private SSTRecord sstRecord;
    private FormatTrackingHSSFListener formatListener;

    /**
     * So we known which sheet we're on
     */
    private int sheetIndex = -1;
    private BoundSheetRecord[] orderedBSRs;
    private ArrayList boundSheetRecords = new ArrayList();

    // For handling formulas with string results
    private int nextRow;
    private int nextColumn;
    private boolean outputNextStringRecord;
    private String d = "";

    private final static String OUTPUT_CHARSET = "UTF-8";
    //    private final String OUTPUT_CHARSET = "GBK";
    private int count = 0;

    public static String deal_time = "";

    /**
     * Creates a new XLS -> CSV converter
     *
     * @param fs         The POIFSFileSystem to process
     * @param output     The PrintStream to output the CSV to
     * @param minColumns The minimum number of columns to output, or -1 for no minimum
     */
    public XlsToCsv(POIFSFileSystem fs, PrintStream output, int minColumns) {
        this.fs = fs;
        this.output = output;
        this.minColumns = minColumns;
    }

    public XlsToCsv(String inputFilePath, String outputFilePath) throws Exception {
        fs = new POIFSFileSystem(new FileInputStream(inputFilePath));
        output = new PrintStream(outputFilePath, OUTPUT_CHARSET);
        minColumns = -1;
    }

    /**
     * Creates a new XLS -> CSV converter
     *
     * @param filename   The file to process
     * @param minColumns The minimum number of columns to output, or -1 for no minimum
     * @throws IOException
     * @throws FileNotFoundException
     */
    public XlsToCsv(String filename, int minColumns) throws IOException, FileNotFoundException {
        this(new POIFSFileSystem(new FileInputStream(filename)), System.out, minColumns);
    }

    public XlsToCsv() {
    }

    public XlsToCsv(long time_cha) {
        this.time_cha = time_cha;
    }

    /**
     * Initiates the processing of the XLS file to CSV
     */
    public void process() throws IOException {
        MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this);
        formatListener = new FormatTrackingHSSFListener(listener);
        HSSFEventFactory factory = new HSSFEventFactory();
        HSSFRequest request = new HSSFRequest();
        if (outputFormulaValues) {
            request.addListenerForAllRecords(formatListener);
        } else {
            workbookBuildingListener = new EventWorkbookBuilder.SheetRecordCollectingListener(formatListener);
            request.addListenerForAllRecords(workbookBuildingListener);
        }
        factory.processWorkbookEvents(request, fs);
    }


    /**
     * Main HSSFListener method, processes events, and outputs the CSV as the
     * file is processed.
     */
    public void processRecord(Record record) {
        int thisRow = -1;
        int thisColumn = -1;
        String thisStr = null;
        String sign = "";
        switch (record.getSid()) {
            case BoundSheetRecord.sid:
                //boundSheetRecords.add(record);
                break;
            case BOFRecord.sid:
                BOFRecord br = (BOFRecord) record;
                if (br.getType() == BOFRecord.TYPE_WORKSHEET) {
                    // Create sub workbook if required
                    if (workbookBuildingListener != null && stubWorkbook == null) {
                        stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook();
                    }

                    // Output the worksheet name
                    // Works by ordering the BSRs by the location of
                    // their BOFRecords, and then knowing that we
                    // process BOFRecords in byte offset order
                    sheetIndex++;
                    if (orderedBSRs == null) {
                        orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords);
                    }
                }
                break;

            case SSTRecord.sid:
                sstRecord = (SSTRecord) record;
                break;

            case BlankRecord.sid:
                BlankRecord brec = (BlankRecord) record;
                thisRow = brec.getRow();
                thisColumn = brec.getColumn();
//                thisStr = "";
                break;
            case BoolErrRecord.sid:
                BoolErrRecord berec = (BoolErrRecord) record;
                thisRow = berec.getRow();
                thisColumn = berec.getColumn();
//                thisStr = "";
                break;

            case FormulaRecord.sid:
                FormulaRecord frec = (FormulaRecord) record;
                thisRow = frec.getRow();
                thisColumn = frec.getColumn();
                if (outputFormulaValues) {
                    if (Double.isNaN(frec.getValue())) {
                        // Formula result is a string
                        // This is stored in the next record
                        outputNextStringRecord = true;
                        nextRow = frec.getRow();
                        nextColumn = frec.getColumn();
                    } else {
                        thisStr = formatListener.formatNumberDateCell(frec);
                    }
                } else {
//                    thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"';
                    thisStr = HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression());
                }
                break;
            case StringRecord.sid:
                if (outputNextStringRecord) {
                    // String for formula
                    StringRecord srec = (StringRecord) record;
                    thisStr = srec.getString();
                    thisRow = nextRow;
                    thisColumn = nextColumn;
                    outputNextStringRecord = false;
                }
                break;

            case LabelRecord.sid:
                LabelRecord lrec = (LabelRecord) record;

                thisRow = lrec.getRow();
                thisColumn = lrec.getColumn();
//                thisStr = '"' + lrec.getValue() + '"';
                thisStr = lrec.getValue();
                break;
            case LabelSSTRecord.sid:
                LabelSSTRecord lsrec = (LabelSSTRecord) record;
                thisRow = lsrec.getRow();
                thisColumn = lsrec.getColumn();
                if (sstRecord == null) {
                    thisStr = '"' + "(No SST Record, can't identify string)" + '"';
//                } else if (sstRecord.getString(lsrec.getSSTIndex()).toString().contains("30/03/2018 - 16h34:18.440")) {
                } else if (sstRecord.getString(lsrec.getSSTIndex()).toString().contains("/")) {
//                    thisStr = '"' + sstRecord.getString(lsrec.getSSTIndex()).toString() + '"';
                    thisStr = sstRecord.getString(lsrec.getSSTIndex()).toString();
                    //用-分割得到时间数据的年月日和小时，分钟秒的数据
                    String[] splitData = thisStr.split("-");
                    //格式化年月日的数据
                    String[] date = splitData[0].split("/");
                    //格式化小时分钟秒的数据
                    String[] time = splitData[1].split(":");
                    String[] hourAndMount = time[0].split("h");
                    String hours = hourAndMount[0] + ":" + hourAndMount[1] + ":" + time[1].split("\\.")[0] + "." + time[1].split("\\.")[1];
                    d = date[date.length - 1].trim() + "-" + date[1] + "-" + date[0];
                    deal_time = d + " " + hours;
                    thisStr = "";
                } else {
                    //处理时间格式
                    thisStr = sstRecord.getString(lsrec.getSSTIndex()).toString();
//                    System.out.println("------->" + thisStr);
                    if (thisStr.contains("h")) {
                        String deal_get_time_split[] = thisStr.split(":");
                        String deal_hour_minute[] = deal_get_time_split[0].split("h");
                        String deal_seconds_millis[] = deal_get_time_split[1].split(",");
//                        String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1];
                        String deal_date = d + " " + deal_hour_minute[0] + ":" + deal_hour_minute[1] + ":" + deal_seconds_millis[0] + "." + deal_seconds_millis[1];
//                        thisStr =  d + " " + deal_hour_minute_ + ":" + deal_seonds_millis_;
                        try {
                            long t = SystemConstants.stdMSsdf.parse(deal_date).getTime() + this.getTime_cha();
                            thisStr = SystemConstants.stdMSsdf.format(new Date(t));
                            System.out.println("s===" + thisStr);
                        } catch (ParseException e) {
                            e.printStackTrace();
                        }
                    } else if (thisStr.contains("Sign") || thisStr.contains("N2")) {
                        //处理信号的信号名。包括信号产生时间和对应的值
                        if (thisStr.contains("N2")) {
                            thisStr = thisStr;
                        } else {
                            sign += thisStr + "_time," + thisStr + "_value";
                            thisStr = sign;
                        }

                    } else {
                        thisStr = "";
                    }

                }
                break;
            case NoteRecord.sid:
                NoteRecord nrec = (NoteRecord) record;
                thisRow = nrec.getRow();
                thisColumn = nrec.getColumn();
                thisStr = '"' + "(TODO)" + '"';
                break;
            case NumberRecord.sid:
                NumberRecord numrec = (NumberRecord) record;
                thisRow = numrec.getRow();
                thisColumn = numrec.getColumn();
                // Format
                thisStr = formatListener.formatNumberDateCell(numrec);
                break;
            case RKRecord.sid:
                RKRecord rkrec = (RKRecord) record;
                thisRow = rkrec.getRow();
                thisColumn = rkrec.getColumn();
                thisStr = '"' + "(TODO)" + '"';
                break;
            default:
                break;
        }
        // Handle new row
        if (thisRow != -1 && thisRow != lastRowNumber) {
            lastColumnNumber = -1;
        }
        // Handle missing column
        if (record instanceof MissingCellDummyRecord) {
            MissingCellDummyRecord mc = (MissingCellDummyRecord) record;
            thisRow = mc.getRow();
            thisColumn = mc.getColumn();
            thisStr = "";
        }

        // If we got something to print out, do so
        if (thisStr != null) {
            if (thisStr == "") {
            } else {
//                System.out.println("thisColumn :" + thisColumn + " thisStr: " + thisStr);
                if (thisColumn > 0) {
                    //控制那一列不被写入到csv文件中，具体以thisStr来决定
                    if ((thisColumn - 1) % 4 == 0) {
                        thisStr = "";

                    } else {
                        output.print(',');
                    }
                }
                output.print(thisStr);
            }

        }

        // Update column and row count
        if (thisRow > -1) {
            lastRowNumber = thisRow;
        }
        if (thisColumn > -1) {
            lastColumnNumber = thisColumn;
        }

        // Handle end of row
        if (record instanceof LastCellOfRowDummyRecord) {
            if (((LastCellOfRowDummyRecord) record).getRow() > 7) {
                // Print out any missing commas if needed
                if (minColumns > 0) {
                    // Columns are 0 based
                    if (lastColumnNumber == -1) {
                        lastColumnNumber = 0;
                    }
                    for (int i = lastColumnNumber; i < (minColumns); i++) {
                        output.print(',');
                    }
                }
                // We're onto a new row
                lastColumnNumber = -1;
                // End the row
                output.println();
            }
        }

    }

    public static void main(String[] args) throws Exception {
        long star = System.currentTimeMillis();
        System.out.println("开始读取的时间是======》::   " + star);
        String inputPath2 = "/home/java/data/fault_database/20180330-第一次试验-试验阶段.xls";
        String outputPath2 = "/home/java/data/fault_database/data.csv";
//        XlsToCsv.time_cha=2000;
        XlsToCsv xls2csv = new XlsToCsv(inputPath2, outputPath2);
        xls2csv.setTime_cha(2000);
        xls2csv.process();
        System.out.println(XlsToCsv.deal_time);
        long end = System.currentTimeMillis();
        System.out.println("总过用时==========>: " + (end - star));
    }


}

​

import com.monitorjbl.xlsx.StreamingReader;
import jxl.WorkbookSettings;
import jxl.read.biff.BiffException;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;

import java.io.*;
import java.util.*;

/**
 * Created by 李泽华 on 2017/9/7.
 */
public class BigExcelReaderPoi {
    private static OutputStream os;
    private static OutputStreamWriter osw;
    private static BufferedWriter bw;
    private static WorkbookSettings ws = new WorkbookSettings();
    private static jxl.Workbook wk;
    private static FileInputStream in;



    /**
     * xls 后缀的execl表格文件内容转成csv文件
     * (1)使用流的方法读取execl表格内容，然后处理表格的内容
     * (2)获得第2行数据，处理得到该文件生成的时间
     * (3)处理第4行数据，获取信号名，然后把信号名分别加上"_time"和"_value"
     * (4)把处理好的数据按照csv文件格式要求写入文件中，在解析车台数据用用到生成的csv文件
     * @param inputPath
     * @param outputPath
     * @return
     */
    public static List> xlsConvertToCsv(String inputPath, String outputPath) {
        List> startTime_outFilePath = new ArrayList>();
        try {
            long start_time = System.currentTimeMillis();
            System.out.println("当前开始时间是=====>:" + start_time);
            os = new FileOutputStream(new File(outputPath));
            osw = new OutputStreamWriter(os, "UTF8");
            bw = new BufferedWriter(osw);
            //载入excel文件
            ws.setLocale(new Locale("en", "EN"));
            wk = jxl.Workbook.getWorkbook(new File(inputPath), ws);
            //从工作簿workbook取得每页sheets
            String date_ = "";
            boolean flag = true;
            String deal_time = "";
            String sign_date = "";
            String sign ="";
            StringBuffer signBuffer = new StringBuffer();
            String startTime = "";
            Map outFilePath_map = new HashMap();
            outFilePath_map.put("outFilePath",outputPath);
            startTime_outFilePath.add(outFilePath_map);
            for (int sheet = 0; sheet < wk.getNumberOfSheets(); sheet++) {
                jxl.Sheet s = wk.getSheet(sheet);
                jxl.Cell[] row = null;
                //从每页sheet取得每个区块cell
                for (int i = 0; i < s.getRows(); i++) {
                    row = s.getRow(i);
                    //获取开始时间 如：2018-03-30
                    if (i == 2){
                        String rowData = row[1].getContents();
                        //用-分割得到时间数据的年月日和小时，分钟秒的数据
                        String[] splitData = rowData.split("-");
                        //格式化年月日的数据
                        String[] date = splitData[0].split("/");

                        //格式化小时分钟秒的数据
                        String[] time = splitData[1].split(":");
                        String[] hourAndMount = time[0].split("h");
                        String hours = hourAndMount[0] + ":" + hourAndMount[1] + ":" + time[1].split("\\.")[0] + "." + time[1].split("\\.")[1];
                        deal_time = date[date.length - 1].trim() + "-" + date[1] + "-" + date[0];
                        startTime = deal_time + " " + hours;


                    }

                    //处理信号的名的值处理后的值如：
                    // Sign1_time,Sign1_value,Sign2_time,Sign2_value,Sign3_time,Sign3_value,Sign4_time,Sign4_value,Sign5_time,Sign5_value,Sign6_time,Sign6_value,Sign7_time,Sign7_value,
                    if (i == 4){
                        for (int m = 0;m8)&&(row.length > 0)) {
                        //写入第一列处理后的数据，并加上,
                        String time = "";
                        String deal_get_time_split[] = row[0].getContents().split(":");
                        String deal_hour_minute[] = deal_get_time_split[0].split("h");
                        String deal_hour_minute_ = deal_hour_minute[0] + ":" + deal_hour_minute[1];
                        String deal_seconds_millis[] = deal_get_time_split[1].split(",");
                        String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1];
                        time = deal_time + " " + deal_hour_minute_ + ":" + deal_seonds_millis_;
                        bw.write(time);
                        bw.write(',');
                        bw.write(row[2].getContents());
                        bw.write(',');
                        for (int j = 1; j < row.length; j++) {
//                            bw.write(',');
                            //取出时间并处理然后写入新的文件中
                            if (j % 4 ==0){
                                String ti ="";
                                String deal_get_time_split_[] = row[j].getContents().split(":");
                                String deal_hour_minute1[] = deal_get_time_split_[0].split("h");
                                String deal_hour_minute_1 = deal_hour_minute1[0] + ":" + deal_hour_minute1[1];
                                String deal_seconds_millis1[] = deal_get_time_split_[1].split(",");
                                String deal_seonds_millis_1 = deal_seconds_millis1[0] + "." + deal_seconds_millis1[1];
                                ti = deal_time + " " + deal_hour_minute_1 + ":" + deal_seonds_millis_1;
                                bw.write(ti);
                                bw.write(',');
                                //获得信号的值
                                bw.write(row[j+2].getContents());
                                bw.write(',');
                            }
                        }
                        bw.newLine();

                    }

                }
            }
            Map  startTime_map = new HashMap();
            startTime_map.put("startTime", startTime);
            startTime_outFilePath.add(startTime_map);
            long end_time = System.currentTimeMillis();
            System.out.println("用时=======>: " + (end_time -start_time) + "ms");
            bw.flush();
            bw.close();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (BiffException e) {
            e.printStackTrace();
        }
        return startTime_outFilePath;
    }


    /**
     * .xlsx类型的execl表格数据处理成为csv文件
     * @param inputPath
     * @param outputPath
     * @return
     */
    public String  xlsxConvertToCsv(String inputPath, String outputPath) {


        long startTime = System.currentTimeMillis();
        System.out.println("程序开始时间：" + (startTime) + "ms");
        try {
            os = new FileOutputStream(new File(outputPath));
            osw = new OutputStreamWriter(os, "UTF8");
            bw = new BufferedWriter(osw);
            in = new FileInputStream(inputPath);
            Workbook wk = StreamingReader.builder()
                    .rowCacheSize(100)  //缓存到内存中的行数，默认是10
                    .bufferSize(4096)  //读取资源时，缓存到内存的字节大小，默认是1024
                    .open(in);  //打开资源，必须，可以是InputStream或者是File，注意：只能打开XLSX格式的文件
            Sheet sheet = wk.getSheetAt(0);
            bw.write(sheet.getSheetName());
            bw.newLine();
            //控制前几行不被写入csv文件
            int count = 0;
            //控制那些列不被写入csv文件
            int cell_index = 0;
            //遍历所有的行
            for (Row row : sheet) {
                //System.out.println("开始遍历第" + row.getRowNum() + "行数据：");
                count++;
                System.out.println("当前行数是==============》： " + count);
                if (count < 8) {
                    continue;
                }
                //遍历所有的列
                for (Cell cell : row) {
                    cell_index++;
                    System.out.println("当前的列号是==========》： " + cell.getColumnIndex());
                    String line_data = "";
                    // System.out.print(cell.getStringCellValue() + " ");
                    if (cell.getStringCellValue().contains("h")) {
                        String deal_get_time_split[] = cell.getStringCellValue().split(":");
                        String deal_hour_minute[] = deal_get_time_split[0].split("h");
                        String deal_hour_minute_ = deal_hour_minute[0] + ":" + deal_hour_minute[1];
//                        String deal_seconds_millis[] = deal_get_time_split[1].split(",");
                        String deal_seconds_millis[] = deal_get_time_split[1].split("\\.");
                        String deal_seonds_millis_ = deal_seconds_millis[0] + "." + deal_seconds_millis[1];
                        line_data += deal_hour_minute_ + ":" + deal_seonds_millis_;
                        bw.write(line_data);
                        bw.write(',');
                    } else if ((cell.getColumnIndex() + 1) % 2 == 0) {
                        bw.write(line_data);
//                        bw.write(',');
                    } else {
                        bw.write(cell.getStringCellValue());
                        bw.write(',');
                    }

                }
                bw.newLine();
                //System.out.println(" ");
            }
            long endTime = System.currentTimeMillis();
            System.out.println("程序运行时间：" + (endTime - startTime) + "ms");    //输出程序运行时间
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                os.close();
                osw.close();
                bw.close();
                in.close();
                wk.close();
            } catch (IOException e) {
                e.printStackTrace();
            }


        }
        return outputPath;
    }

    //TEST
    public static void main(String args[]) {

//        String inputPath2 = "/home/java/data/fault_database/20180330-第一次试验-试验阶段.xls";
        String inputPath2 = "d:/home/java/data/fault_database/20180330-第一次试验-试验阶段.xls";
//        String outputPath2 = "/home/java/data/fault_database/data.csv";
        String outputPath2 = "d:/home/java/data/fault_database/fault_databasedata.csv";
        String csv = "/home/java/data/fault_database/lizehua_test.csv";
        xlsConvertToCsv(inputPath2, outputPath2);
    }
}