springboot项目 实现CSV文件百万数据入库

需要先引入解析工具

		
		
			com.opencsv
			opencsv
			4.3.2
		

抽取出工具类

package io.renren.utils;

import com.opencsv.bean.CsvToBean;
import com.opencsv.bean.CsvToBeanBuilder;
import com.opencsv.bean.HeaderColumnNameMappingStrategy;
import org.springframework.web.multipart.MultipartFile;

import java.io.InputStreamReader;
import java.util.List;

public class cvsUtils {
    public static  List getCsvData(MultipartFile file, Class clazz) {
        InputStreamReader in = null;
        try {
            in = new InputStreamReader(file.getInputStream(), "gbk");
        } catch (Exception e) {
            System.out.println("读取文件失败");
        }

        HeaderColumnNameMappingStrategy strategy = new HeaderColumnNameMappingStrategy<>();
        strategy.setType(clazz);

        CsvToBean csvToBean = new CsvToBeanBuilder(in)
                .withSeparator(',')
                .withQuoteChar('\'')
                .withMappingStrategy(strategy).build();
        return csvToBean.parse();
    }


}

案例demo

package io.renren.entity.csv;

import com.opencsv.bean.CsvBindByName;


public class User {


    @CsvBindByName(column = "id")
    private Long id;


    /**
     * 字典代码
     */
    @CsvBindByName(column = "code")
    private String code;

    /**
     * 简写
     */
    @CsvBindByName(column = "shortName")
    private String shortName;

    /**
     * N
     * 名称
     */
    @CsvBindByName(column = "name")  //是否可以为null 否
    private String name;


    public String getCode() {
        return code;
    }

    public void setCode(String code) {
        this.code = code;
    }

    public String getShortName() {
        return shortName;
    }

    public void setShortName(String shortName) {
        this.shortName = shortName;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }


    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }


}

下面是核心代码
使用线程池和java8Stream流数据分片去处理

配置好线程池 和 CompletableFuture 去做异步处理
线程池配置

package io.renren.config;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.ThreadPoolExecutor;

@Configuration
@EnableAsync
public class ThreadPoolTaskExecutorConfig {
    @Bean("taskModuleExecutor")
    ThreadPoolTaskExecutor getCrawler1(){
        ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor();
        threadPoolTaskExecutor.setCorePoolSize(5);
        threadPoolTaskExecutor.setMaxPoolSize(6);
        threadPoolTaskExecutor.setQueueCapacity(200);
        threadPoolTaskExecutor.setThreadNamePrefix("task-concurrent-work");
        threadPoolTaskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        threadPoolTaskExecutor.initialize();
        return threadPoolTaskExecutor;
    }

}```

//每一千条批量插入 （这里根据自己的需求去定义）
public static  final long PAGE_SIZE =1000;

@PostMapping("/uploadVCSBeach")
public R uploadVCSBeachThere(MultipartFile file) throws IOException {



    //cvs工具类解析数据
    List userList = cvsUtils.getCsvData(file, User.class);

    if (CollectionUtils.isNotEmpty(userList)){

        //总页数
        long count = userList.size()/PAGE_SIZE;

        //遍历每一页
        for (Integer i = 1; i < count+1; i++) {
            int n =i;
		
		// 这里是做分片
        List subUserList = userList.stream()
                .skip((n - 1) * PAGE_SIZE) //方法用于跳过前n条数据
                .limit(PAGE_SIZE) //方法用于返回前n条数据，
                .collect(Collectors.toList());
            try {
                CompletableFuture voidCompletableFuture = CompletableFuture.runAsync(() -> {

                    // 批量插入
                    userDao.insertBatch(subUserList);
                }, threadPoolExecutor);
             //   CompletableFuture.allOf(voidCompletableFuture).get();

            } catch (Exception e) {
                e.printStackTrace();
            }

        }


    }

    return R.ok();
}