SpringBatch 批处理分区(Partitioner )分片(九)

前言：　在Springbatch中，Partitioning意味着对数据进行分片，然后每片实现专门处理，假设单线程处理100个数据需要10分钟，但是 我们将100个数据分片成十块，每块单独处理，时间可能只需要1分钟。

SpringBatch其它文章直通车：

• SpringBatch读单个文件(FlatFileItemReader)和写单个文件(FlatFileItemWriter)(一)
• SpringBatch顺序读取多文件(MultiResourceItemReader)和顺序写文件(MultiResourceItemWriter)(二)
• SpringBatch读数据库(MyBatisPagingItemReader)(三)
• SpringBatch读文件(FlatFileItemReader)写据库(MyBatisBatchItemWriter)(四)
• SpringBatch 监听器之Job监听器(JobExecutionListener)和Step监听器(StepExecutionListener)(五)
• SpringBatch 监听器之Chunk监听器(ChunkListener)和Skip监听器(SkipListener)(六)
• SpringBatch 多线程(TaskExecutor)启动Job详解 (七)
• SpringBatch 配置并行启动Job详解 (八)

代码已上传GitHub上面地址：https://github.com/FadeHub/spring-boot-learn/tree/master/spring-boot-springbatch

分区原理是一个master主处理器对应多个从slave处理器官方原图为：

官方原文处理器的节点可以是远程服务器的服务，也可以是本地执行的线程。主处理器发送给从处理器的消息是不需要持久或实现JMS那种严格的保证消息传递的，Spring Batch元数据JobRepository会确保每个slave执行一次，每次Job执行只执行一次。

• 本章先以本地分区举例，需求为从数据库cat表中导出10个csv文件

一、cat表数据准备

1、cat实体类

package com.sl.entity;

/**
 * @author shuliangzhao
 * @Title: Cat
 * @ProjectName spring-boot-learn
 * @Description: TODO
 * @date 2019/9/8 14:29
 */
public class Cat {

    private Integer id;

    private String catname;

    private String catage;

    private String cataddress;

    public Integer getId() {
        return id;
    }

    public void setId(Integer id) {
        this.id = id;
    }

    public String getCatname() {
        return catname;
    }

    public void setCatname(String catname) {
        this.catname = catname;
    }

    public String getCatage() {
        return catage;
    }

    public void setCatage(String catage) {
        this.catage = catage;
    }

    public String getCataddress() {
        return cataddress;
    }

    public void setCataddress(String cataddress) {
        this.cataddress = cataddress;
    }
}

2、数据库表cat和数据

CREATE TABLE `cat` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `catname` varchar(20) DEFAULT NULL,
  `catage` varchar(3) DEFAULT NULL,
  `cataddress` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=45 DEFAULT CHARSET=utf8mb4;
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (5, '大黄', '1', '上丰一路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (6, '小黄', '2', '上丰二路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (7, '咖啡猫', '4', '上丰三路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (8, '折耳猫', '1', '上丰四路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (9, '大黄', '1', '上丰一路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (10, '小黄', '2', '上丰二路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (11, '咖啡猫', '4', '上丰三路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (12, '折耳猫', '1', '上丰四路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (13, '大黄', '1', '上丰一路');
INSERT INTO `cat`(`id`, `catname`, `catage`, `cataddress`) VALUES (14, '小黄', '2', '上丰二路');

3、application.properties配置文件

spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.url=jdbc:mysql://localhost/chapter01?useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
spring.datasource.username=root
spring.datasource.password=root


mybatis.type-aliases-package=com.sl.entity
mybatis.mapperLocations=classpath:mapper/*.xml

二、分区catPartitionerJob配置

1、分区reader

package com.sl.common;

import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.batch.MyBatisPagingItemReader;

import java.util.HashMap;
import java.util.Map;

/**
 * 分区读取数据库
 * @author shuliangzhao
 * @Title: CommonMybatisItemReader
 * @ProjectName spring-boot-learn
 * @Description: TODO
 * @date 2019/9/15 20:14
 */
public class CommonPartitionMybatisItemReader extends MyBatisPagingItemReader {

    public CommonPartitionMybatisItemReader(SqlSessionFactory sqlSessionFactory, String name,String fromId,String toId) {
        setSqlSessionFactory(sqlSessionFactory);
        setQueryId("com.sl.entity."+name+".selectPartitionList");
        Map parameterValues = new HashMap<>();
        parameterValues.put("fromId", fromId);
        parameterValues.put("toId", toId);
        setParameterValues(parameterValues);
        setPageSize(100);
    }
}

2、分区writer

package com.sl.common;

import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor;
import org.springframework.batch.item.file.transform.DelimitedLineAggregator;
import org.springframework.core.io.FileSystemResource;

import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.List;

/**
 * 公共写
 * @author shuliangzhao
 * @Title: CommonFileItemWriter
 * @ProjectName spring-boot-learn
 * @Description: TODO
 * @date 2019/9/15 20:48
 */
public class CommonPartitionFileItemWriter extends FlatFileItemWriter {

    private FileSystemResource fileSystemResource;

    public void init(Class clz) {
        BeanWrapperFieldExtractor beanWrapperFieldExtractor = new BeanWrapperFieldExtractor();
        Field[] fields = clz.getDeclaredFields();
        List list = new ArrayList<>();
        for (Field field : fields) {
            if (!Modifier.isStatic(field.getModifiers())) {
                list.add(field.getName());
            }
        }
        String[] names = new String[list.size()];
        beanWrapperFieldExtractor.setNames(list.toArray(names));
        beanWrapperFieldExtractor.afterPropertiesSet();
        DelimitedLineAggregator lineAggregator = new DelimitedLineAggregator();
        lineAggregator.setDelimiter(",");
        lineAggregator.setFieldExtractor(beanWrapperFieldExtractor);
        setLineAggregator(lineAggregator);
        setName(clz.getSimpleName());
        setEncoding(CommonConstants.ENCODING_READ);
    }

    public CommonPartitionFileItemWriter(Class clz,String fromId,String toId) {
        init(clz);
        fileSystemResource = new FileSystemResource("D:\\aplus\\shuqian\\source\\"+ clz.getSimpleName()+"-"+fromId + "-" + toId  + ".csv");
        setResource(fileSystemResource);

    }
}

3、分区processor

package com.sl.processor;

import com.sl.common.CommonProcessor;
import com.sl.entity.CafeCat;
import com.sl.entity.Cat;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.stereotype.Component;

/**
 * @author shuliangzhao
 * @Title: CatProcessor
 * @ProjectName spring-boot-learn
 * @Description: TODO
 * @date 2019/9/15 20:13
 */
@Component
@StepScope
public class CatPartitionProcessor extends CommonProcessor {

    @Override
    public void processor(CafeCat o, Cat cat) {
        o.setCataddress(cat.getCataddress());
        o.setCatage(cat.getCatage());
        o.setCatname(cat.getCatname());
    }
}

4、CatPartitioner分区

package com.sl.partition;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.stereotype.Component;

import java.util.HashMap;
import java.util.Map;

/**
 * @author shuliangzhao
 * @Title: CatPartitioner
 * @ProjectName spring-boot-learn
 * @Description: TODO
 * @date 2019/9/15 20:00
 */
@Component
public class CatPartitioner implements Partitioner {

    private static final Logger log = LoggerFactory.getLogger(CatPartitioner.class);

    @Override
    public Map partition(int gridSize) {
        log.info("partition  gridsize is " + gridSize);
        Map result = new HashMap<>();
        int range = 10;
        int fromId = 1;
        int toId = range;
        for (int i = 1; i <= gridSize; i++) {
            ExecutionContext value = new ExecutionContext();

            log.info("\nStarting : Thread" + i);
            log.info("fromId : " + fromId);
            log.info("toId : " + toId);

            value.putInt("fromId", fromId);
            value.putInt("toId", toId);

            // give each thread a name, thread 1,2,3
            value.putString("name", "Thread" + i);

            result.put("partition" + i, value);

            fromId = toId + 1;
            toId += range;

        }
        return result;
    }
}

5、job配置

package com.sl.config;

import com.sl.common.CommonPartitionFileItemWriter;
import com.sl.common.CommonPartitionMybatisItemReader;
import com.sl.entity.CafeCat;
import com.sl.entity.Cat;
import com.sl.partition.CatPartitioner;
import com.sl.processor.CatPartitionProcessor;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.partition.PartitionHandler;
import org.springframework.batch.core.partition.support.TaskExecutorPartitionHandler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;

/**
 * 分区job
 * @author shuliangzhao
 * @Title: CatPartitionerJob
 * @ProjectName spring-boot-learn
 * @Description: TODO
 * @date 2019/9/15 19:56
 */
@Configuration
@EnableBatchProcessing
public class CatMasterPartitionerJob {

    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    @Autowired
    private StepBuilderFactory stepBuilderFactory;

    @Autowired
    private CatPartitioner catPartitioner;

    @Autowired
    private SqlSessionFactory sqlSessionFactory;

    @Autowired
    private CatPartitionProcessor catPartitionProcessor;


    @Bean   
    public Job catPartitionerJob() {
         return jobBuilderFactory.get("catPartitionerJob")
                 .start(catMasterStep())
                 .build();
    }

    @Bean
    public Step catMasterStep() {
        return stepBuilderFactory.get("catMasterStep").partitioner(catSlaveStep().getName(),catPartitioner)
                .partitionHandler(catPartitionHandler()).build();
    }

    @Bean
    public PartitionHandler catPartitionHandler() {
        TaskExecutorPartitionHandler handler = new TaskExecutorPartitionHandler();
        handler.setGridSize(10);
        handler.setTaskExecutor(catPartitionHandlerTaskExecutor());
        handler.setStep(catSlaveStep());
        try {
            handler.afterPropertiesSet();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return handler;
    }

    @Bean
    public SimpleAsyncTaskExecutor catPartitionHandlerTaskExecutor() {
        return new SimpleAsyncTaskExecutor();
    }

    @Bean
    public Step catSlaveStep() {
        return stepBuilderFactory.get("catSlaveStep")
                .chunk(10)
                .reader(commonPartitionMybatisItemReader(null,null))
                .processor(catPartitionProcessor)
                .writer(commonPartitionFileItemWriter(null,null))
                .build();
    }

    @Bean
    @StepScope
    public CommonPartitionMybatisItemReader commonPartitionMybatisItemReader( @Value("#{stepExecutionContext[fromId]}") final String fromId,
                                                                              @Value("#{stepExecutionContext[toId]}") final String toId) {
        return new CommonPartitionMybatisItemReader(sqlSessionFactory, Cat.class.getSimpleName(),fromId,toId);
    }

    @Bean
    @StepScope
    public CommonPartitionFileItemWriter commonPartitionFileItemWriter(@Value("#{stepExecutionContext[fromId]}") final String fromId,
                                                                       @Value("#{stepExecutionContext[toId]}") final String toId) {
         return new CommonPartitionFileItemWriter(CafeCat.class,fromId,toId);
    }
}

执行结果生成10个文件

以上代码就是简单实现了本地分区，当然还有不足。如果使用时请认真测试，满足自己的需求。