excel导入并解析入库,多线程实现
此demo涉及多线程,远程调用,excel从页面读取解析(用流的方式读);先从页面选择excel, 然后点击导入按钮, 后台用POI解析校验, 然后远程调用ejb,存入数据库.(注意:excel文件并没有存储在服务器.)框架是基于struts1, 里面的多线程实现方式是实现Callable接口.
public ActionForward importUleAddress(ActionMapping mapping,
ActionForm form, HttpServletRequest request,
HttpServletResponse response){
response.setContentType("text/html;charset=utf-8");
Map resultMap = new HashMap();
//默认启动的线程数量
int threaPoolNum = 10;
long starTime=System.currentTimeMillis();
//创建定长线程池
ExecutorService espool = Executors.newFixedThreadPool(threaPoolNum);
//创建校验数据线程序列集合
// ArrayList checkDataThreadList = new ArrayList();
//创建插入数据线程序列集合
ArrayList insertUleThreadList = new ArrayList();
//接收返回值
List>> futureList = new ArrayList>>();
List> uleLists = new ArrayList>();
violationCancelInfoForm proForm = (violationCancelInfoForm) form;
//为每一个线程创建独立的参数
for (int i = 0; i < threaPoolNum; i++) {
uleLists.add(i, new ArrayList());
}
log.info("------------ start read file!------------:");
try {
InputStream input = proForm.getCsvFile().getInputStream();
String excelUrl = proForm.getCsvFile().getFileName();
boolean isE2007 = false; //判断是否是excel2007格式
if(excelUrl.endsWith("xlsx") || excelUrl.endsWith("XLSX")){
isE2007 = true;
}
// 根据文件格式(2003或者2007)来初始化
Workbook wb = isE2007 ? new XSSFWorkbook(input) : new HSSFWorkbook(input);
Sheet sheet = wb.getSheetAt(0); //获得第一个表单
Iterator rows = sheet.rowIterator(); //获得第一个表单的迭代器
//存放excel表格中的code用于判断是否有重复code
List insertList = new ArrayList();
//读取文件中的数据放入insertList
//当前行数
int rowNum = 0;
while (rows.hasNext()) {
Row row = rows.next(); //获得行数据
rowNum = row.getRowNum();
//第一行无效数据，跳过
//校验数据有无
Map checkResultMap = checkData(row);
if(checkResultMap!=null){
AddressMerchantForm uleAddress = new AddressMerchantForm();
uleAddress = (AddressMerchantForm) checkResultMap.get("uleAddress");
if(uleAddress!=null){
insertList.add(uleAddress);
}
if(checkResultMap.get("uleAddress") != null){
if(rowNumP00==0){
log.info("------------ 当前已解析"+rowNum+"条!------------");
}
}
}
}
long endTime=System.currentTimeMillis();
long readFileTime=endTime-starTime;
log.info("------------ 校验通过!开始插入！------------");
//启动线程插入数据
int num = 0;
//为插入数据的每个线程的list设置数据,每个线程处理一个uleLists
for(AddressMerchantForm uleAddress : insertList){
for(int i=0; i
if (num % threaPoolNum == i) {
uleLists.get(i).add(uleAddress);
}
}
num++;
}
//将list分给每个线程
for (int i = 0; i < uleLists.size(); i++) {
if (uleLists.get(i).size() > 0 && uleLists.get(i) !=null && !uleLists.get(i).equals("")) {
insertUleThreadList.add(new InsertUleThread(uleLists.get(i)));
}
}
int insertNum = 0;
//启动插入数据线程
if (insertUleThreadList.size() > 0 && insertUleThreadList != null) {
futureList = espool.invokeAll(insertUleThreadList);
insertUleThreadList.clear();
for (int i = 0; i < uleLists.size(); i++) {
if(uleLists.get(i)!=null){
//统计excel数据条数
insertNum += uleLists.get(i).size();
}
uleLists.get(i).clear();
}
for (Future> future : futureList) {
Map map = future.get();
if (!map.isEmpty()) {
if("1".equals(map.get("status"))){
log.info("------------ 数据成功插入!------------count:"+insertNum+" "+map.get("tName"));
resultMap.put("importMsg", "成功插入数据"+insertNum+"条！");
}else{
log.info("------------ 数据插入失败!------------count:"+insertNum+" "+map.get("tName"));
resultMap.put("importMsg", "数据插入失败!");
}
}
}
}
//解析完最后文件内容就关闭线程池
espool.shutdown();
endTime=System.currentTimeMillis();
endTime=endTime-starTime;
log.info("------------ readFileTime:------------"+readFileTime+"ms");
log.info("------------ time:------------"+endTime+"ms");
} catch (Exception e) {
log.error("------------ error!------------e: "+e);
System.out.println(e);
resultMap.put("importMsg", "插入数据异常！");
}
//导入excel后的结果信息返回
try {
response.getWriter().write(resultMap.get("importMsg").toString());
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
//校验表格中必要列数据的缺失情况
public Map checkData(Row row){
Map map = new HashMap();
int rownum = row.getRowNum();
if(rownum == 0){
if(!ExcelPOIUtil.isCellBlank(row.getCell(0)) && !ExcelPOIUtil.isCellBlank(row.getCell(1)) && !ExcelPOIUtil.isCellBlank(row.getCell(2))
&& !ExcelPOIUtil.isCellBlank(row.getCell(3)) && !ExcelPOIUtil.isCellBlank(row.getCell(4)) && !ExcelPOIUtil.isCellBlank(row.getCell(5))
&& !ExcelPOIUtil.isCellBlank(row.getCell(6)) && !ExcelPOIUtil.isCellBlank(row.getCell(7)) && !ExcelPOIUtil.isCellBlank(row.getCell(8))
&& !ExcelPOIUtil.isCellBlank(row.getCell(9))){
if("*邮乐地址code".equals(ExcelPOIUtil.getValue(row.getCell(0)))
&& "*邮乐地址名称".equals(ExcelPOIUtil.getValue(row.getCell(1)))
&& "*父编码".equals(ExcelPOIUtil.getValue(row.getCell(2)))
&& "*等级编码(数字)".equals(ExcelPOIUtil.getValue(row.getCell(3)))
&& "*外源名".equals(ExcelPOIUtil.getValue(row.getCell(4)))
&& "*三方地址code".equals(ExcelPOIUtil.getValue(row.getCell(5)))
&& "*三方名称".equals(ExcelPOIUtil.getValue(row.getCell(6)))
){
map.put("uleAddress", null);
}else{
log.info("------------ 模板被修改！ ------------");
map.put("importMsg", "请下载最新模板！");
return map;
}
}else{
log.info("------------ 模板有空！ ------------");
map.put("importMsg", "请下载最新模板！");
return map;
}
}else{
AddressMerchantForm uleAddress = new AddressMerchantForm();
if(!ExcelPOIUtil.isCellBlank(row.getCell(0))){
uleAddress.setUleAddressCode(ExcelPOIUtil.getValue(row.getCell(0)));
}else{
log.info("------------ ule code缺失！ ------------第"+rownum+"行");
map.put("importMsg", "第" + rownum + "行uleCode缺失！");
return map;
}
if(!ExcelPOIUtil.isCellBlank(row.getCell(1))){
uleAddress.setUleAddressName(ExcelPOIUtil.getValue(row.getCell(1)));
}else{
log.info("------------ ule name缺失！ ------------第"+rownum+"行");
map.put("importMsg", "第" + rownum + "行uleName缺失！");
return map;
}
if(!ExcelPOIUtil.isCellBlank(row.getCell(2))){
uleAddress.setUleParentCode(ExcelPOIUtil.getValue(row.getCell(2)));
}else{
log.info("------------ ule 父code缺失！ ------------第"+rownum+"行");
map.put("importMsg", "第" + rownum + "行uleParentCode缺失");
return map;
}
if(!ExcelPOIUtil.isCellBlank(row.getCell(3))){
uleAddress.setAddressLevel(ExcelPOIUtil.getValue(row.getCell(3)));
}else{
log.info("------------ addressLevel缺失！ ------------第"+rownum+"行");
map.put("importMsg", "第" + rownum + "行addressLevel缺失");
return map;
}
if(!ExcelPOIUtil.isCellBlank(row.getCell(4))){
uleAddress.setSourceFlag(ExcelPOIUtil.getValue(row.getCell(4)));
}else{
log.info("------------ sourceFlag缺失！ ------------第"+rownum+"行");
map.put("importMsg", "第" + rownum + "行sourceFlag缺失");
return map;
}
if(!ExcelPOIUtil.isCellBlank(row.getCell(5))){
uleAddress.setTpAddressCode(ExcelPOIUtil.getValue(row.getCell(5)));
}else{
log.info("------------ tpAddressCode缺失！ ------------第"+rownum+"行");
map.put("importMsg", "第" + rownum + "行tpAddressCode缺失！");
return map;
}
if(!ExcelPOIUtil.isCellBlank(row.getCell(6))){
uleAddress.setTpAddressName(ExcelPOIUtil.getValue(row.getCell(6)));
}else{
log.info("------------ tpAddressName缺失！ ------------第"+rownum+"行");
map.put("importMsg", "第" + rownum + "行tpAddressName缺失！");
return map;
}
map.put("uleAddress", uleAddress);
}
return map;
}
--------------------------------------------------------------------
package com.ule.cs.csmerchant.struts.action.address;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import com.ule.cs.csmerchant.utils.Config;
import com.ule.cs.csmerchant.utils.JsonUtil;
public class InsertUleThread implements Callable> {
private Logger log = Logger.getLogger(InsertUleThread.class);
private Map map = new HashMap();
String httpUrl=Config.getProperty("addressMapping.batchInsert");
int num = 1;//插入成功
int failNum = 0; //插入失败
public InsertUleThread (List list){
Map map = new HashMap();
map.put("list", list);
this.map = map;
}
@Override
public Map call() throws Exception {
//插入
try {
// 调用HTTPClient
System.out.println(map);
Map result=PostAddressInfo(null ,null , map , httpUrl,"batchInsert");
log.info("addressMapping.batchInsert result is :>>>>>>"+result);
log.info("addressMapping.batchInsert end..........");
map.put("count", num);
map.put("status", "1");
map.put("tName", Thread.currentThread().getName());
if(!"0000".equals(result.get("returnCode").toString())){
map.put("count", 0);
map.put("status","0");
map.put("tName", Thread.currentThread().getName());
}
} catch (Exception e) {
log.error("----error----"+e);
map.put("count", failNum);
map.put("status","0");
map.put("tName", Thread.currentThread().getName());
}
return map;
}
public Map PostAddressInfo(Integer currentPage,Integer pageSize, Map map,String httpUrl,String flag){
HttpClient httpclient = new DefaultHttpClient();
Map tMap = new HashMap();
httpclient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 600000);
httpclient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT, 600000);
System.out.println(httpUrl);
try {
HttpPost httpPost = new HttpPost(httpUrl);
List nvps = new ArrayList();
if(flag.equals("del") || flag.equals("queryByID")){
Iterator iterator = map.entrySet().iterator();
while(iterator.hasNext()){
Entry elem = (Entry) iterator.next();
nvps.add(new BasicNameValuePair(elem.getKey(),elem.getValue()));
}
}else if(flag.equals("insert") || flag.equals("update")){
JSONObject jsonObject = JSONObject.fromObject(map);
nvps.add(new BasicNameValuePair("param_json",jsonObject.toString()));
}else if(flag.equals("batchInsert")){
JSONArray jsonObject = JSONArray.fromObject(map.get("list"));
nvps.add(new BasicNameValuePair("param_json",jsonObject.toString()));
System.out.println("-----------------------------------------"+nvps);
}else{
JSONObject jsonObject = JSONObject.fromObject(map);
nvps.add(new BasicNameValuePair("param_json",jsonObject.toString()));
nvps.add(new BasicNameValuePair("currentPage",currentPage.toString()));
nvps.add(new BasicNameValuePair("pageSize",pageSize.toString()));
}
httpPost.setEntity(new UrlEncodedFormEntity(nvps, "UTF-8"));
HttpResponse httpResponse = httpclient.execute(httpPost);
HttpEntity entity = httpResponse.getEntity();
String httpResult = "" ;
if (entity != null) {
httpResult = EntityUtils.toString(entity);
entity.consumeContent();
}
tMap = JsonUtil.jsonToObject(httpResult, Map.class);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
return tMap;
}
}
-----------------------------------------------------------------
excel模板如下: