Java使用poi读取excel数据转换成csv再读取，通过数据库配置信息插入数据
一、首先配置相关maven包
org.apache.poi
poi
4.1.0
org.apache.poi
poi-ooxml
4.1.0
org.apache.commons
commons-lang3
3.9
com.alibaba
fastjson
1.2.62
二、解析处理大数据Excel文件XLSX2CSV.java类
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.poi.ooxml.util.SAXHelper;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.openxml4j.opc.PackageAccess;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.util.CellAddress;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler;
import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler.SheetContentsHandler;
import org.apache.poi.xssf.model.StylesTable;
import org.apache.poi.xssf.usermodel.XSSFComment;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
/**
* Created with IntelliJ IDEA.
* Description: 读取文件数据
*/
public class XLSX2CSV {
/**
* Uses the XSSF Event SAX helpers to do most of the work
* of parsing the Sheet XML, and outputs the contents
* as a (basic) CSV.
*/
private class SheetToCSV implements SheetContentsHandler {
private boolean firstCellOfRow = false;
private int currentRow = -1;
private int currentCol = -1;
private void outputMissingRows(int number) {
for (int i = 0; i < number; i++) {
curStr = new ArrayList();
for (int j = 0; j < minColumns; j++) {
curStr.add(null);
}
output.add(curStr);
}
}
@Override
public void startRow(int rowNum) {
curStr = new ArrayList();
// If there were gaps, output the missing rows
outputMissingRows(rowNum - currentRow - 1);
// Prepare for this row
firstCellOfRow = true;
currentRow = rowNum;
currentCol = -1;
}
@Override
public void endRow(int rowNum) {
// Ensure the minimum number of columns
for (int i = currentCol; i < minColumns ; i++) {
curStr.add(null);
}
output.add(curStr);
}
@Override
public void cell(String cellReference, String formattedValue,
XSSFComment comment) {
// gracefully handle missing CellRef here in a similar way as XSSFCell does
if (cellReference == null) {
cellReference = new CellAddress(currentRow, currentCol).formatAsString();
}
// Did we miss any cells?
int thisCol = (new CellReference(cellReference)).getCol();
int missedCols = thisCol - currentCol - 1;
for (int i = 0; i < missedCols; i++) {
curStr.add(null);
}
currentCol = thisCol;
// Number or string?
try {
Double.parseDouble(formattedValue);
curStr.add(formattedValue);
} catch (NumberFormatException e) {
// output.append('"');
curStr.add(formattedValue);
// output.append('"');
}
}
@Override
public void headerFooter(String text, boolean isHeader, String tagName) {
// Skip, no headers or footers in CSV
}
}
private final OPCPackage xlsxPackage;
/**
* Number of columns to read starting with leftmost
*/
private final int minColumns;
/**
* Destination for data
*/
private ArrayList> output;
private ArrayList curStr;
public ArrayList> getOutput(){
return output;
}
/**
* Creates a new XLSX -> CSV converter
* @param pkg The XLSX package to process
* @param minColumns The minimum number of columns to output, or -1 for no minimum
*/
public XLSX2CSV(OPCPackage pkg, int minColumns) {
this.xlsxPackage = pkg;
this.minColumns = minColumns;
}
/**
* Parses and shows the content of one sheet
* using the specified styles and shared-strings tables.
*
* @param styles
* @param strings
* @param sheetInputStream
*/
public void processSheet(
StylesTable styles,
ReadOnlySharedStringsTable strings,
SheetContentsHandler sheetHandler,
InputStream sheetInputStream)
throws ParserConfigurationException, SAXException {
DataFormatter formatter = new DataFormatter();
InputSource sheetSource = new InputSource(sheetInputStream);
try {
XMLReader sheetParser = SAXHelper.newXMLReader();
ContentHandler handler = new XSSFSheetXMLHandler(
styles, null, strings, sheetHandler, formatter, false);
sheetParser.setContentHandler(handler);
sheetParser.parse(sheetSource);
} catch (ParserConfigurationException e) {
throw new RuntimeException("SAX parser appears to be broken - " + e.getMessage());
} catch (IOException io) {
io.printStackTrace();
}
}
/**
* Initiates the processing of the XLS workbook file to CSV.
*/
public ArrayList> process(String sheetName) {
try{
ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(this.xlsxPackage);
XSSFReader xssfReader = new XSSFReader(this.xlsxPackage);
StylesTable styles = xssfReader.getStylesTable();
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader.getSheetsData();
while (iter.hasNext()) {
output = new ArrayList> ();
InputStream stream = iter.next();
if(iter.getSheetName().equals(sheetName)){
processSheet(styles, strings, new SheetToCSV(), stream);
close(stream);
return output;
}
close(stream);
continue;
}
} catch (OpenXML4JException open) {
open.printStackTrace();
} catch (ParserConfigurationException parser) {
parser.printStackTrace();
} catch (SAXException sax) {
sax.printStackTrace();
} catch (IOException io) {
io.printStackTrace();
}
return null;
}
private void close(InputStream stream){
try {
stream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
File xlsxFile = new File("F:\\files\\test.xlsx");
if (!xlsxFile.exists()) {
System.err.println("Not found or not a file: " + xlsxFile.getPath());
return;
}
// The package open is instantaneous, as it should be.
OPCPackage p = null;
try {
p = OPCPackage.open(xlsxFile.getPath(), PackageAccess.READ);
} catch (InvalidFormatException e) {
e.printStackTrace();
}
String[] array = {"序号","名称","年龄","性别","班级"};
XLSX2CSV xlsx2csv = new XLSX2CSV(p, array.length);
xlsx2csv.process("测试");
ArrayList> data = xlsx2csv.getOutput();
if (!checkFormat(data.get(0),array)) {
System.out.println("文档标题格式不正确!");
}
try {
p.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 校验excel表头是否符合规范
* @param list
* @param array
* @return
*/
private static boolean checkFormat(ArrayList list,String[] array){
if (list == null || array == null) {
return false;
}
String[] data = list.toArray(new String[list.size()]);
for(int i = 0; i < array.length; i++){
if (!data[i].equals(array[i])) {
return false;
}
}
return true;
}
}
三、数据库配置表excel_sheet，主要根据配置读取excel文档指定sheet页，根据配置获取对应列的数据插入到数据库表中
表中数据sort可排序，读取时使用该值-1 相当于该列在excel文档的sheet页中属于第几列，使用required字段值判断是否需要该列的数据插入数据库
四、根据数据库配置表获取excel数据的写法
/**
* 读取EXCEL数据并入库
*/
public void addExcel() throws Exception {
//本地文件读取
String fileName = "E:\\文档\\测试数据v1.0.xlsx";
File file = new File(fileName);
// 通过前端传输MultipartFile文件流处理方式
// OPCPackage p = OPCPackage.open(file.getInputStream());
OPCPackage p = OPCPackage.open(file);
List excels = excelSheetMapper.getExcelSheet("CSCES", null);
List list = new ArrayList<>();
for (int i = 0; i < excels.size(); i++) {
String sheetName = excels.get(i).getSheetName();
int minColumns = excels.get(i).getColumns().size();
XLSX2CSV xlsx2csv = new XLSX2CSV(p, minColumns);
ArrayList> data = xlsx2csv.process(sheetName);
if (data != null && data.size() > 0) {
List json = readArray(data, excels.get(i).getColumns());
List object = JSONUtils.jsonToList(json.toString(), TestData.class);
list.addAll(object);
}
}
if (list != null && list.size() > 0) {
excelSheetMapper.insertTestData(list);
}
}
private List readArray(ArrayList> data,
List columns) {
List list = new ArrayList<>();
for (int i = 1; i < data.size(); i++) {
JSONObject object = new JSONObject();
ArrayList strings = data.get(i);
for (int j = 0; j < columns.size(); j++) {
ExcelSheetEntity sheet = columns.get(j);
if (sheet.getRequired() == 1) {
int index = sheet.getSort() - 1;
object.put(sheet.getAttributeName(), strings.get(index));
}
}
list.add(object);
}
return list;
} 相关实体类ExcelEntity、ExcelSheetEntity
package json.tool.entity.model;
import java.io.Serializable;
import java.util.List;
/**
* Created with IntelliJ IDEA.
* Description:
* Date: 2019/10/16
* Time: 15:58
*/
public class ExcelEntity implements Serializable {
private String typeName;
private String sheetName;
private List columns;
public String getTypeName() {
return typeName;
}
public void setTypeName(String typeName) {
this.typeName = typeName;
}
public String getSheetName() {
return sheetName;
}
public void setSheetName(String sheetName) {
this.sheetName = sheetName;
}
public List getColumns() {
return columns;
}
public void setColumns(List columns) {
this.columns = columns;
}
}
package json.tool.entity.model;
import java.io.Serializable;
/**
* Created with IntelliJ IDEA.
* Description:
* Date: 2019/10/16
* Time: 15:59
*/
public class ExcelSheetEntity implements Serializable {
private String attributeName;
private String attributeValue;
private int sort;
private int required;
public String getAttributeName() {
return attributeName;
}
public void setAttributeName(String attributeName) {
this.attributeName = attributeName;
}
public String getAttributeValue() {
return attributeValue;
}
public void setAttributeValue(String attributeValue) {
this.attributeValue = attributeValue;
}
public int getSort() {
return sort;
}
public void setSort(int sort) {
this.sort = sort;
}
public int getRequired() {
return required;
}
public void setRequired(int required) {
this.required = required;
}
}
Mapper.xml文件
INSERT INTO test_data (
id,pattern_number,c_name,data_number,data_name,
type_str,fact_table,parent_number,field_number,field_name,
field_type,`length`,edit_mode,dropdown_mask,value_set,data_type,
required,remark,`time`
)
VALUES
(
#{item.id},#{item.patternNumber},#{item.cName},#{item.dataNumber},
#{item.dataName},#{item.typeStr},#{item.factTable},#{item.parentNumber},
#{item.fieldNumber},#{item.fieldName},#{item.fieldType},#{item.length},
#{item.editMode},#{item.dropdownMask},
#{item.valueSet},#{item.dataType},
#{item.required},#{item.remark},#{item.time}
)
JSON转换成bean工具类
package json.tool.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONException;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
/**
* Created with IntelliJ IDEA.
* Description:
* Date: 2019-08-31
* Time: 14:22
*/
public class JSONUtils {
/**
* bean对象转换成JSON字符串
* @param object
* @return
*/
public static String beanToJson(Object object) {
try{
if (object != null) {
return JSON.toJSONString(object);
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* JSON字符串转换成bean对象
* @param json
* @param clazz
* @return
*/
public static T jsonToBean(String json, Class clazz) {
try{
if (StringUtils.isNotBlank(json) && clazz != null) {
return JSON.parseObject(json, clazz);
}
} catch (JSONException e) {
e.printStackTrace();
}
return null;
}
/**
* List转换成JSON字符串
* @param clazz
* @param
* @return
*/
public static String listToJson(List clazz) {
try {
return JSON.toJSONString(clazz);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* JSON字符串转换成List
* @param json
* @param clazz
* @param
* @return
*/
public static List jsonToList(String json, Class clazz) {
try {
if (StringUtils.isNotBlank(json) && clazz != null) {
return JSON.parseArray(json, clazz);
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}