一次导入2w行的表格，并通过注解校验字段

背景

最近项目中有一个需求，需要一次通过excel导入大量的数据。之前用的easypoi测试需要大约一小时才能完成，这样明显是不行的。
深入了解之后，其实只需要将批式处理改为流式处理便可以解决解析慢的问题。综合考虑后选用了easyexcel这个框架。
测试2W行的数据从导入到入库大约需要5s,大大加快了速度。 后续增加了HibernateValidator进行校验数据。

代码

web导入：

@RestController
@RequestMapping("easyexcel")
public class EasyExcelTest  {
    @Autowired
    private ExecutiveInfoService executiveInfoService;

    @PostMapping("import_excel")
   public Object importExcel(@RequestParam("file") MultipartFile file) throws IOException {
        UploadDataListener uploadDataListener = new UploadDataListener(executiveInfoService);
        EasyExcel.read(file.getInputStream(), ExecutiveInfoExcel.class,  uploadDataListener).sheet().doRead();
        // 这里根据自己的返回类型去修改
        if (uploadDataListener.getInsertFlag()){
            return "success";
        }
        return uploadDataListener.getFailList();
    }

    @GetMapping("download")
    public void download(HttpServletResponse response) throws IOException {
        // 这里注意 有同学反应使用swagger 会导致各种问题，请直接用浏览器或者用postman
        response.setContentType("application/vnd.ms-excel");
        response.setCharacterEncoding("utf-8");
        // 这里URLEncoder.encode可以防止中文乱码 当然和easyexcel没有关系
        String fileName = URLEncoder.encode("zms", "UTF-8");
        response.setHeader("Content-disposition", "attachment;filename=" + fileName + ".xlsx");
        EasyExcel.write(response.getOutputStream(), ExecutiveInfoExcel.class).sheet("模板").doWrite(null);
    }
}

解析表格DTO:

@Data
public class ExecutiveInfoExcel  {

    @ExcelProperty("零售客户名")
    @NotBlank
    private String executiveName;
    @ExcelProperty("企业名称")
    @NotBlank
    private String enterpriseName;
    @ExcelProperty("统一社会信用代码")
    @NotNull(message = "不能为空" )
    private String enterpriseCreditCode;
    @ExcelProperty("详细地址")
    @NotBlank
    private String enterpriseAddress;
    @ExcelProperty("零售客户号")
    @NotNull
    private Long executiveId;
    @ExcelProperty("零售客户身份")
    @NotBlank
    private String executivePosition;
    @ExcelProperty("注册资金")
    @NotBlank
    private String registerCapital;
    @ExcelProperty("法定代表人")
    @NotBlank
    private String legalRepresentative;
    @ExcelProperty("电话")
    @NotBlank
    private String telephone;
    @ExcelProperty("邮箱")
    @NotBlank
    private String email;
    @ExcelProperty("公司类型")
    @NotBlank
    private String companyType;
    @ExcelProperty("所属行业")
    @NotBlank
    private String industry;
    @ExcelProperty("卡号")
    @NotNull
    private Integer cardNumber;
    @ExcelProperty("身份证号码")
    @NotBlank
    private String idCard;
    @ExcelProperty("客户号开立日期")
    @DateTimeFormat("yyyy-MM-dd")
    private Date customerOpeningDay;
    @ExcelProperty("联系电话")
    @NotNull
    private Integer telephoneNumber;
    @ExcelProperty("成立时间")
    @DateTimeFormat("yyyy-MM-dd")
    private Date establishmentTime;
    @ExcelProperty("最高持卡级别")
    @NotBlank
    private String highestCardholderLevel;
    @ExcelProperty("是否办理过小微贷款业务")
    @NotBlank
    private String flagMicroLoanBusiness;

}

解析监听类

public class  UploadDataListener extends AnalysisEventListener<ExecutiveInfoExcel> {
    private static final Logger LOGGER =
            LoggerFactory.getLogger(UploadDataListener.class);

    private ExecutiveInfoService executiveInfoService;
    private Boolean insertFlag = true;
    private List<Integer> failList = new ArrayList<>();

    List<ExecutiveInfoExcel> list = new ArrayList<ExecutiveInfoExcel>();

    /**
     * 如果使用了spring,请使用这个构造方法。每次创建Listener的时候需要把spring管理的类传进来
     *
     * @param
     */
    public UploadDataListener(ExecutiveInfoService executiveInfoService) {
        this.executiveInfoService = executiveInfoService;
    }

    /**
     * 这个每一条数据解析都会来调用
     *
     * @param data
     *            one row value. Is is same as {@link #()}
     * @param context
     */
    @Override
    public void invoke(ExecutiveInfoExcel data, AnalysisContext context) {
        //LOGGER.info("解析到一条数据:{}", JSON.toJSONString(data));
        Validator validator = HibernateValidator.getValidator();
        Set<ConstraintViolation<ExecutiveInfoExcel>> validateSet = validator.validate(data,Default.class);
        if (validateSet != null && !validateSet.isEmpty()) {
            //System.out.println("校验出错：" + data);
            LOGGER.info("校验出错: {}",data);
            failList.add(context.readRowHolder().getRowIndex());
            insertFlag = false;
        }else {
            list.add(data);
        }

    }

    /**
     * 所有数据解析完成了 都会来调用
     *
     * @param context
     */
    @Override
    public void doAfterAllAnalysed(AnalysisContext context) {
        // 这里也要保存数据，确保最后遗留的数据也存储到数据库
       if (insertFlag){
           saveData();
       }else {
           LOGGER.info("有校验错误不入库");
       }
    }

    @Override
    public void onException(Exception exception, AnalysisContext context){
        LOGGER.error("exception: {}",exception.toString());
        context.currentReadHolder();
    }

    @Override
    public void invokeHead(Map<Integer, CellData> var1, AnalysisContext var2){

    }

    /**
     * 加上存储数据库
     */
    private void saveData() {
       LOGGER.info("{}条数据，开始存储数据库！", list.size());
        List<ExecutiveInfoEntity> entityList = ConvertUtils.sourceToTarget(list, ExecutiveInfoEntity.class);
        executiveInfoService.insert(entityList);
       // LOGGER.info("存储数据库成功！");
    }

    public List<Integer> getFailList() {
        return failList;
    }

    public Boolean getInsertFlag() {
        return insertFlag;
    }
}

异步插入数据库，具体的实体类这里省略

  public void insert(List<ExecutiveInfoEntity> entityList) {
        //一次插入3000条数据到数据库
        int num = 1;
        asynRedisService.testzms(entityList);
        int toIndex=num;
        int listSize = entityList.size();
        for (int i=0; i< entityList.size(); i+=num){
            if (i+num > listSize){
                toIndex = listSize-i;
            }
            List<ExecutiveInfoEntity> infoEntities = entityList.subList(i, i+toIndex);
            LOGGER.info("入库{}", i);
            executiveInfoDao.insertOrUpdate(infoEntities);
        }
    }

开启校验工具类

public class HibernateValidator {
    private static Validator validator = Validation.byProvider( org.hibernate.validator.HibernateValidator.class )
            .configure()
            .addProperty( "hibernate.validator.fail_fast", "true" )
            .buildValidatorFactory().getValidator();

    private HibernateValidator() {

    }

    public static Validator getValidator() {
        return validator;
    }
}

类型转换工具类

public class ConvertUtils {
    private static Logger logger = LoggerFactory.getLogger(ConvertUtils.class);

    public static <T> T sourceToTarget(Object source, Class<T> target){
        if(source == null){
            return null;
        }
        T targetObject = null;
        try {
            targetObject = target.newInstance();
            BeanUtils.copyProperties(source, targetObject);
        } catch (Exception e) {
            logger.error("convert error ", e);
        }

        return targetObject;
    }

    public static <T> List<T> sourceToTarget(Collection<?> sourceList, Class<T> target){
        if(sourceList == null){
            return null;
        }

        List targetList = new ArrayList<>(sourceList.size());
        try {
            for(Object source : sourceList){
                T targetObject = target.newInstance();
                BeanUtils.copyProperties(source, targetObject);
                targetList.add(targetObject);
            }
        }catch (Exception e){
            logger.error("convert error ", e);
        }

        return targetList;
    }
}

项目地址：git地址