多线程处理百万级数据入库

一、使用Thread线程+JDBC批处理入库

1. 创建线程类重写run方法
   

2. 设置线程名称（判断该线程是否结束）
   

3. jdbc批处理处理数据

实例代码

/**
	 * 批量新增产品
	 * @param set
	 */
	public int batchAddProduct(List<RegisteredProduct> set){
		if(0 < set.size()) {
			System.out.println("新增产品开始执行++++++++" + LocalTime.now());
			String insertSql = "insert into registered_product(storage_location_id,brand_id,product_category_id,product_location_type) values(?, ?, ?, ?)";
			java.sql.Connection conn = null;
			PreparedStatement stmt = null;
			try {
				conn = SessionFactoryUtils.getDataSource(getSessionFactory()).getConnection();
				stmt = conn.prepareStatement(insertSql, Statement.RETURN_GENERATED_KEYS);
				// 方式2：批量提交
				conn.setAutoCommit(false);
				for (int i = 0; i < set.size(); i++) {
					// 产品信息
					RegisteredProduct product = set.get(i);
					if (null != product.getStorageLocation() && null != product.getStorageLocation().getStorageLocationId()) {
						stmt.setLong(1, product.getStorageLocation().getStorageLocationId());
					} else {
						stmt.setNull(1, Types.INTEGER);
					}
					if (null != product.getBrand() && null != product.getBrand().getBrandId()) {
						stmt.setLong(2, product.getBrand().getBrandId());
					} else {
						stmt.setNull(2, Types.INTEGER);
					}
					Long aLong = null != product.getProductCategory() ? product.getProductCategory().getProductCategoryId() : 4L;
					stmt.setLong(3, aLong);
					stmt.setString(4, !StringUtil.isEmpty(product.getProductLocationType()) ? product.getProductLocationType() : "");
					stmt.addBatch();
					if (i > 0 && i % 500 == 0) {
						stmt.executeBatch();
						conn.commit();
					}
				}
				stmt.executeBatch();
				conn.commit();
			} catch (SQLException e) {
				// TODO 自动生成的 catch 块
				e.printStackTrace();
			} finally {
				try {
					if (stmt != null) stmt.close();
					if (conn != null) conn.close();
				} catch (SQLException e) {
					// TODO 自动生成的 catch 块
					e.printStackTrace();
				}
			}
		}
		return 1;
	}

4. 结束线程

二、使用线程池分段处理list集合

SplitListUtils:

 public static <T> List<List<T>> splitList(List<T> list, int len) {
        if (list == null || list.size() == 0 || len < 1) {
            return Lists.newArrayList();
        }
        List<List<T>> result = Lists.newArrayList();
        int size = list.size();
        int count = (size + len - 1) / len;
        for (int i = 0; i < count; i++) {
            List<T> subList = list.subList(i * len, ((i + 1) * len > size ? size : len * (i + 1)));
            result.add(subList);
        }
        return result;
    }

实例代码：