EasyExcel多线程批量导出数据，动态表头，静态资源访问

1.导入依赖

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>easyexcel</artifactId>
            <version>3.1.1</version>
        </dependency>

2.建立实体

@Data
public class ActResultLogVO implements Serializable {
    private static final long serialVersionUID = 1L;

    @ExcelProperty(value = "onlineseqid",index = 0)
    private String onlineseqid;

    @ExcelProperty(value = "businessid",index = 1)
    private String businessid;

    @ExcelProperty(value = "becifno",index = 2)
    private String becifno;

    @ExcelProperty(value = "ivisresult",index = 3)
    private String ivisresult;

    @ExcelProperty(value = "createdby",index = 4)
    private String createdby;

    @ExcelProperty(value = "createddate",index = 5)
    private LocalDate createddate;

    @ExcelProperty(value = "updateby",index = 6)
    private String updateby;

    @ExcelProperty(value = "updateddate",index = 7)
    private LocalDate updateddate;

    @ExcelProperty(value = "risklevel",index = 8)
    private String risklevel;
    /**
     * 分页开始数量
     */
    private Integer start;
    /**
     *  分页 size
     */
    private Integer size;
}

3.多线程异步导出

	@Async
	@Transactional
    public  void exportExcel(String filePath) {
    	//filePath 文件路径
        ExcelWriter writer = EasyExcel.write(filePath, ActResultLogVO.class).build();
        // 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1
        int N = 2;
        // 大小设置为2就可以，作为缓冲
        BlockingQueue<List<ActResultLogVO>> queue = new ArrayBlockingQueue<>(2);
        AtomicInteger start = new AtomicInteger(0);
        AtomicInteger num = new AtomicInteger(0);
        AtomicInteger sheet = new AtomicInteger(1);
        // 分页大小可以适当调整
        int pageSize = 100000;
        //开启多个线程分页查数据
        for (int i = 0; i < N; i++) {
            executorService.submit(() -> {
                while (true) {
                    //自增
                    int startNum= start.getAndAdd(pageSize);
                    try {
                        long l = System.currentTimeMillis();
                        logger.info("[多线程，分页查询] 线程：{}，开始执行查询：startNum :{},时间：{}",Thread.currentThread().getName(),startNum,l);
                        List<ActResultLogVO> list = selectActResultLogVOPage(startNum, pageSize);
                        logger.info("[多线程，分页查询] 线程：{}，执行查询完成 startNum :{} 用时：{}",Thread.currentThread().getName(),startNum,System.currentTimeMillis()-l);
                        if (CollectionUtils.isEmpty(list)) {
                            //读到没数据也要放入空集合
                            queue.put(Collections.EMPTY_LIST);
                            break;
                        }
                        queue.put(list);
                    } catch (Exception e) {
                        //异常情况也要放入空集合，防止写线程无法退出循环
                        try {
                            queue.put(Collections.EMPTY_LIST);
                        } catch (InterruptedException ex) {
                            throw new RuntimeException(ex);
                        }
                        throw new RuntimeException(e);
                    }
                }
            });
        }
        Future<?> submit = executorService.submit(() -> {
            int count = 0;
            while (true) {
                List<ActResultLogVO> list = null;
                try {
                    list = queue.take();
                } catch (InterruptedException e) {
                    Thread.interrupted();
                }
                if (CollectionUtils.isEmpty(list)) {
                    count++;
                    // 当获取到两次空集合时，说明已经读完
                    if (count == N) {
                        break;
                    }
                    continue;
                }
                if(num.get()>=500000){
                    sheet.getAndAdd(1);
                    num.getAndSet(0);
                }
                num.getAndAdd(list.size());
                long l = System.currentTimeMillis();
                logger.info("[多线程，写入Excel] 线程：{}，开始执行写入：sheet :{},num:{}时间：{}",Thread.currentThread().getName(),sheet.get(),num.get(),l);
                writer.write(list, EasyExcel.writerSheet("Sheet"+sheet).build());
                logger.info("[多线程，写入Excel] 线程：{}，开始执行写入结束：sheet :{},num:{},时间：{}",Thread.currentThread().getName(),sheet.get(),num.get(),System.currentTimeMillis()-l);
            }
            writer.finish();
        });
        try {
            // 阻塞等待完成，异步处理也可以去掉这段代码
            submit.get();
        } catch (Exception e) {
        }
    }

4.动态表头

 	 @Autowired
    @Qualifier("excelThreadPool")
    private ExecutorService executorService;
 
 	@Async
    @Transactional
    public void exportExcel（String filePath,LinkedHashMap<String, DynamicExcelData> nameMap) {
        // 指定写入的文件
        ExcelWriter writer = EasyExcel.write(filePath).head(ExcelUtils.getHead(nameMap)).build();
        // 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1
        int N = 2;
        // 大小设置为2就可以，作为缓冲
        BlockingQueue<List<Map<String,Object>>> queue = new ArrayBlockingQueue<>(2);
        AtomicInteger num = new AtomicInteger(0);
        AtomicInteger sheet = new AtomicInteger(1);
     
        // 分页大小可以适当调整
        int pageSize = ExcelConstants.SELECT_TO_DB_ROWS_MYBATIS;
        //开启多个线程分页查数据
        getData(lvnengAllCardBillVo,queue,pageSize);
        Future<?> submit = executorService.submit(() -> {
            int count = 0;
            while (true) {
                List<Map<String,Object>> list = null;
                try {
                    list = queue.take();
                } catch (InterruptedException e) {
                    Thread.interrupted();
                }
                if (CollectionUtils.isEmpty(list)) {
                    count++;
                    // 当获取到两次空集合时，说明已经读完
                    if (count == N) {
                        break;
                    }
                    continue;
                }
                if(num.get()>=600000){
                    sheet.getAndAdd(1);
                    num.getAndSet(0);
                }
                num.getAndAdd(list.size());
                List<List<String>> dataList = ExcelUtils.getDataList(nameMap, list);
                writer.write(dataList, EasyExcel.writerSheet("Sheet"+sheet).build());
            }
            writer.finish();
        });
        try {
            // 阻塞等待完成，异步处理也可以去掉这段代码
            submit.get();
        } catch (Exception e) {
        }
    }

public void getData(ActResultLogVO actResultLogVO,BlockingQueue<List<Map<String,Object>>> queue,int pageSize) {
        // 根据数据读写速度来调整，一般来说读的逻辑复杂，比较慢，如果读比写快，这里设为1
        int N = 2;
        AtomicInteger start = new AtomicInteger(0);
        for (int i = 0; i < N; i++) {
            executorService.submit(() -> {
                while (true) {
                    //自增
                    int startNum = start.getAndAdd(pageSize);
                    List<Map<String, Object>> list = new ArrayList<>();
                    try {
                        ActResultLogVO actResultLogVO1 = DeepCopyUtil.deepCopy(actResultLogVO);
                        actResultLogVO1.setStart(startNum);
                        List<Map<String, Object>> list = selectactResultLogVOPage(actResultLogVO1);
                        if (CollectionUtils.isEmpty(list)) {
                            //读到没数据也要放入空集合
                            queue.put(Collections.EMPTY_LIST);
                            break;
                        }
                        queue.put(list);

                    } catch (Exception e) {
                        //异常情况也要放入空集合，防止写线程无法退出循环
                        try {
                            queue.put(Collections.EMPTY_LIST);
                        } catch (InterruptedException ex) {
                            throw new RuntimeException(ex);
                        }
                        throw new RuntimeException(e);
                    }
                }
            });

        }
        logger.info("查询数据完成");
    }

@Configuration
public class ThreadPoolConfig {
    @Bean("excelThreadPool")
    public  ExecutorService buildExcelThreadPool() {
        int cpuNum = Runtime.getRuntime().availableProcessors();
        BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(1000);
        ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("excel-pool-%d").build();
        return new ThreadPoolExecutor(10 * cpuNum, 30 * cpuNum,
                1, TimeUnit.MINUTES, workQueue, threadFactory);
    }
}

public class ExcelConstants {
    public static final Integer PER_SHEET_ROW_COUNT = 100*10000;
    public static final Integer PER_WRITE_ROW_COUNT = 20*10000;
    public static final Integer SHEET_WRITE_ROW_COUNT = 50*10000;
    public static final Integer SELECT_TO_DB_ROWS = 10*10000;
    public static final Integer SELECT_TO_DB_ROWS_MYBATIS = 5*10000;
}

@Data
public class DynamicExcelData {
    //列名
    private String name;
    //默认值
    private String defaultValue;

    public DynamicExcelData(String name, String defaultValue) {
        this.name = name;
        this.defaultValue = defaultValue;
    }
}

5,用到的util

5.1 EasyUtils

public class ExcelUtils {

    public static void dynamicExportByURL(String filePath,
                                     LinkedHashMap<String, DynamicExcelData> nameMap,
                                     List<Map<String, Object>> list,
                                     String sheetName) throws IOException {
        //首先判断是否有数据，没有就返回
        if(CollUtil.isEmpty(list)){
            return;
        }
        //这里的map使用LinkedHashMap，实现字段的顺序功能
        if(nameMap==null){
            throw new RuntimeException("请填写好映射表数据");
        }
        File file = getFile(filePath);
        dynamicExportByURL(file,nameMap,list,sheetName);
    }

    public static void dynamicExport(HttpServletResponse response,
                                     LinkedHashMap<String, DynamicExcelData> nameMap,
                                     List<Map<String, Object>> list,
                                     String sheetName) throws IOException {
        //首先判断是否有数据，没有就返回
        if(CollUtil.isEmpty(list)){
            return;
        }
        //这里的map使用LinkedHashMap，实现字段的顺序功能
        if(nameMap==null){
            throw new RuntimeException("请填写好映射表数据");
        }
        response.setHeader("Access-Control-Expose-Headers", "Content-Disposition");
        response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
        response.setCharacterEncoding("utf-8");
        //使用EasyExcel write进行写出
        EasyExcel.write(response.getOutputStream()).head(getHead(nameMap)).sheet(sheetName).doWrite(getDataList(nameMap,list));
    }


    public static ArrayList<List<String>> getHead(LinkedHashMap<String, DynamicExcelData> nameMap) {
        //获取表头
        ArrayList<List<String>> head = new ArrayList<>();
        for (Map.Entry<String, DynamicExcelData> titleMap : nameMap.entrySet()) {
            DynamicExcelData data = titleMap.getValue();
            head.add(Collections.singletonList(data.getName()));
        }
        return head;
    }

    public static List<List<String>> getDataList(LinkedHashMap<String, DynamicExcelData> nameMap, List<Map<String, Object>> list) {
        //先初始化一下传入
        int size = list.size();
        List<List<String>> dataList = new ArrayList<>();
        for (int i = 0; i < size; i++) {
            dataList.add(new ArrayList<>());
        }
        //数据重组
        for (int i = 0; i < list.size(); i++) {
            Map<String, Object> map = list.get(i);
            List<String> columns = dataList.get(i);
            for (Map.Entry<String, DynamicExcelData> sortNameEntry : nameMap.entrySet()) {
                String key = sortNameEntry.getKey();
                Object value = map.get(key);
                columns.add(value != null ? String.valueOf(value) : sortNameEntry.getValue().getDefaultValue());
            }
        }
        return dataList;
    }

    public static File getFile(String filePath) throws IOException {
        File file = new File(filePath);
        if (!file.exists()) {
            file.createNewFile();
        }
        return file;
    }



    public static void dynamicExportByURL(File file,LinkedHashMap<String, DynamicExcelData> nameMap,  List date,String sheetName) {

        long startTime = 0;
        ExcelWriter excelWriter = null;
        try {
            startTime = System.currentTimeMillis();
            // 获取 sheet 的个数
            int sheetNum = getSheetNum(date.size());
            // 获取每个sheet 写入的次数
            int writeNumPerSheet = getWriteNumPerSheet();
            // 最后一个 sheet 写入的数量
            int writeNumLastSheet = date.size() - (sheetNum - 1) * ExcelConstants.PER_SHEET_ROW_COUNT;
            // 最后一个 sheet 写入的次数
            int writeNumPerLastSheet = writeNumLastSheet % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT : writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT + 1;
            // 指定写入的文件
            excelWriter = EasyExcel.write(file).head(getHead(nameMap)).build();
         
            for (int i = 0; i < sheetNum; i++) {
                String sheet = sheetName + i;
                WriteSheet writeSheet = EasyExcel.writerSheet(i, sheet).build();
                int writeNum = i == sheetNum - 1 ? writeNumPerLastSheet : writeNumPerSheet; // 每个sheet 写入的次数
                int endEndNum = i == sheetNum - 1 ? date.size() : (i + 1) * ExcelConstants.PER_SHEET_ROW_COUNT; // 每个sheet 最后一次写入的最后行数
                for (int j = 0; j < writeNum; j++) {
                    long l = System.currentTimeMillis();
                    int startNum = i * ExcelConstants.PER_SHEET_ROW_COUNT + j * ExcelConstants.PER_WRITE_ROW_COUNT;
                    int endNum = j == writeNum - 1 ? endEndNum : i * ExcelConstants.PER_SHEET_ROW_COUNT + (j + 1) * ExcelConstants.PER_WRITE_ROW_COUNT;
                    excelWriter.write(getDataList(nameMap,date.subList(startNum, endNum)), writeSheet);

                }
            }
        } catch (Exception e) {
     
        } finally {
            // 需要放入 finally 中
            if (excelWriter != null) {
                excelWriter.finish();
            }
        }
    }

    public static void writeExcel(String filePath, Class clazz, List date,String sheetName) throws Exception{
        File file = getFile(filePath);
        long startTime = System.currentTimeMillis();
        // 获取 sheet 的个数
        int sheetNum = getSheetNum(date.size());
        // 获取每个sheet 写入的次数
        int writeNumPerSheet = getWriteNumPerSheet();
        // 最后一个 sheet 写入的数量
        int writeNumLastSheet = date.size() - (sheetNum - 1) * ExcelConstants.PER_SHEET_ROW_COUNT;
        // 最后一个 sheet 写入的次数
        int writeNumPerLastSheet = writeNumLastSheet % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT : writeNumLastSheet / ExcelConstants.PER_WRITE_ROW_COUNT + 1;
        // 指定写入的文件
        ExcelWriter excelWriter = EasyExcel.write(file, clazz).build();

        for (int i = 0; i < sheetNum; i++) {
            String sheet = sheetName + i;
            WriteSheet writeSheet = EasyExcel.writerSheet(i, sheet).build();
            int writeNum = i == sheetNum - 1 ? writeNumPerLastSheet : writeNumPerSheet; // 每个sheet 写入的次数
            int endEndNum = i == sheetNum - 1 ? date.size() : (i + 1) * ExcelConstants.PER_SHEET_ROW_COUNT; // 每个sheet 最后一次写入的最后行数
            for (int j = 0; j < writeNum; j++) {
                long l = System.currentTimeMillis();
                int startNum = i * ExcelConstants.PER_SHEET_ROW_COUNT + j * ExcelConstants.PER_WRITE_ROW_COUNT;
                int endNum = j == writeNum - 1 ? endEndNum : i * ExcelConstants.PER_SHEET_ROW_COUNT + (j + 1) * ExcelConstants.PER_WRITE_ROW_COUNT;
                excelWriter.write(date.subList(startNum, endNum), writeSheet);
            }
        }
        // 需要放入 finally 中
        if (excelWriter != null) {
            excelWriter.finish();
        }
    }

    public static int getSheetNum(int dateSize){
        return dateSize % ExcelConstants.PER_SHEET_ROW_COUNT == 0 ? (dateSize / ExcelConstants.PER_SHEET_ROW_COUNT) : (dateSize / ExcelConstants.PER_SHEET_ROW_COUNT + 1);
    }

    public static int getWriteNumPerSheet(){
        return ExcelConstants.PER_SHEET_ROW_COUNT % ExcelConstants.PER_WRITE_ROW_COUNT == 0 ? (ExcelConstants.PER_SHEET_ROW_COUNT / ExcelConstants.PER_WRITE_ROW_COUNT) : (ExcelConstants.PER_SHEET_ROW_COUNT / ExcelConstants.PER_WRITE_ROW_COUNT + 1);
    }

    public static  List<String> objectToList(Object obj){
        ArrayList<String> list = new ArrayList<>();
        //获取obj类中的所有字段
        Field[] fields = obj.getClass().getDeclaredFields();
        //遍历所有属性
        for (Field field : fields) {
            try {
                field.setAccessible(true);
                list.add(field.get(obj).toString());
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
        return list;
    }


    public static Map<String,Object> objectToMap(Object obj){
        HashMap<String, Object> map = new HashMap<>();
        //获取obj类中的所有字段
        Field[] fields = obj.getClass().getDeclaredFields();
        //遍历所有属性
        for (Field field : fields) {
            try {
                field.setAccessible(true);
                map.put(field.getName(),field.get(obj));
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
        return map;
    }


    /**
     * 通过反射方式将头部作为公共部分进行设置
     */
    public static List<List<String>> setTitles(Class clazz) {
        List<List<String>> titles = new ArrayList<List<String>>();
        for (Field declaredField : clazz.getDeclaredFields()) {
            ExcelProperty annotation = declaredField.getAnnotation(ExcelProperty.class);
            if (null != annotation) {
                titles.add(Arrays.asList(annotation.value()));
            }
        }
        return titles;
    }
}

5.2 深拷贝

public class DeepCopyUtil {

    public static <T extends Serializable> T deepCopy(T object) {
        try {
            ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
            ObjectOutputStream objectOutputStream = new ObjectOutputStream(outputStream);
            objectOutputStream.writeObject(object);
            ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
            ObjectInputStream objectInputStream = new ObjectInputStream(inputStream);
            return (T) objectInputStream.readObject();
        } catch (IOException | ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
    }
}

配置静态资源访问

@Configuration
public class MyStaticConfig extends WebMvcConfigurationSupport {


    @Value("${download.url}")
    private String fileDir;
    @Override
    public void addResourceHandlers(ResourceHandlerRegistry registry) {
        registry.addResourceHandler("/uploads/**").addResourceLocations("classpath:/META-INF/resources/", "classpath:/resources/",
                "classpath:/static/", "classpath:/public/","file:"+fileDir);
    }
}