java百万级数据插入解决方案

背景：使用JdbcTemplate来插入百万数据量，避免内存溢出，批量插入

    DataHashMapExtractor extractor = new DataHashMapExtractor(querySize) {
            @Override
            public void processData(List> list, int times, int n) {
                log.info("第{}次读取{}条完成，共{}条",times,list.size(),n);
                batchSizeUpdate(list,p.getInsertSql(),CommonJdbc.getNamedJdbcTemplate(),batchSize);
            }
    };
    jdbcTemplate.query(sendTasks.getSQL_CONTENT(), new HashMap<>(0),extractor);

import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.ResultSetExtractor;
import org.springframework.jdbc.support.JdbcUtils;

import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * 查询数据并返回字段类型
 * @author lang.zhou
 * @since 2023/1/9 17:42
 */
@Slf4j
public abstract class DataHashMapExtractor implements ResultSetExtractor>> {

    private int batchSize = 1000;

    public DataHashMapExtractor() {
    }
    public DataHashMapExtractor(int batchSize) {
        if(batchSize > 1000){
            this.batchSize = batchSize;
        }
    }

    @Override
    public List> extractData(ResultSet rs) throws SQLException, DataAccessException {
        ResultSetMetaData resultSetMetaData = rs.getMetaData();
        int count = resultSetMetaData.getColumnCount();
        int n = 0;
        int times = 0;

        List> list = new ArrayList<>();
        while(rs.next()){
            Map e = new LinkedHashMap<>(count);
            for (int i = 1; i < count + 1; i++) {
                e.putIfAbsent(JdbcUtils.lookupColumnName(resultSetMetaData, i), JdbcUtils.getResultSetValue(rs, i));
            }
            list.add(e);
            n ++;
            if(list.size() >= batchSize){
                times ++;
                this.processData(list,times,n);
                list.clear();
            }
        }
        if(list.size() > 0){
            times ++;
            this.processData(list,times,n);
            list.clear();
        }
        return new ArrayList<>(0);
    }

    public abstract void processData(List> list, int times, int n);


}

    public void batchSizeUpdate(List> list, String sql,NamedParameterJdbcTemplate namedParameterJdbcTemplate, int batchSize){
        int size = list.size();
        int n = size / batchSize;
        int l = size % batchSize;
        if(l > 0){
            n++;
        }
        log.info("总共分"+n+"次插入");
        for (int i = 0; i < n; i++) {
            int start = i*batchSize;
            int end = (i+1)*batchSize;
            if(end > size){
                end = size;
            }
            batchUpdate(list.subList(start,end),sql, namedParameterJdbcTemplate);
            log.info("第"+(i+1)+"次插入完毕");
        }
    }

    private void batchUpdate(List> list, String sql,NamedParameterJdbcTemplate namedParameterJdbcTemplate){
        Map [] param = new Map[list.size()];
        for(int c= 0;c