springboot多数据源+batch批量上传

最近在学习springbatch，遇到了多个数据库相读写问题，问题类似于从A数据库读取数据，保存到B数据库，

1：先配置好数据库地址

datasource:
    type: com.zaxxer.hikari.HikariDataSource
    url: jdbc:mysql://127.0.0.1:3306/test?allowMultiQueries=true&useUnicode=true&characterEncoding=UTF-
    password: root
    password: roo  
driver-class-name: com.mysql.jdbc.Driver
second-datasource:
    type: com.zaxxer.hikari.HikariDataSource
    url: jdbc:mysql://127.0.0.1:3306/test1?allowMultiQueries=true&useUnicode=true&characterEncoding=UTF-8
    username: root
    password: root

    driver-class-name: com.mysql.jdbc.Driver

2：利用springboot配置数据源

数据源

@Configuration
public class DataSourceConfig {

    @Primary
    @Bean(name = "primaryDataSource")
    @Qualifier("primaryDataSource")
    @ConfigurationProperties(prefix = "spring.datasource")
    public DataSource primaryDatasource() {
        return DataSourceBuilder.create().build();
    }

//    @Primary
    @Bean(name = "secondaryDataSource")
    @Qualifier("secondaryDataSource")
    @ConfigurationProperties(prefix = "spring.second-datasource")
    public DataSource secondaryDataSource() {
        return DataSourceBuilder.create().build();
    }


}

 主数据库配置

@Configuration
@EnableTransactionManagement
@EnableJpaRepositories(
        entityManagerFactoryRef = "entityManagerFactoryPrimary",//配置连接工厂 entityManagerFactory
        transactionManagerRef = "transactionManagerPrimary", //配置 事物管理器  transactionManager
        basePackages = "com.ics.top.repository"//设置dao（repo）所在位置
)
public class PrimaryConfig {
    @Autowired
    private JpaProperties jpaProperties;

    @Autowired
    @Qualifier("primaryDataSource")
    private DataSource primaryDataSource;


    @Primary
    @Bean(name = "entityManagerPrimary")
    public EntityManager entityManagerPrimary(EntityManagerFactoryBuilder builder) {
        EntityManager entityManager = entityManagerFactoryPrimary(builder).getObject().createEntityManager();
        return entityManager;
    }

    /**
     *
     * @param builder
     * @return*/

    @Bean(name = "entityManagerFactoryPrimary")
    @Primary
    public LocalContainerEntityManagerFactoryBean entityManagerFactoryPrimary(EntityManagerFactoryBuilder builder) {
        return builder
                //设置数据源
                .dataSource(primaryDataSource)
                //设置数据源属性
                .properties(getVendorProperties(primaryDataSource))
                //设置实体类所在位置.扫描所有带有 @Entity 注解的类
                .packages("com.ics.top.domain")
                // Spring会将EntityManagerFactory注入到Repository之中.有了 EntityManagerFactory之后,
                // Repository就能用它来创建 EntityManager 了,然后Entity就可以针对数据库执行操作
                .persistenceUnit("primaryPersistenceUnit")
                .build();

    }

    private Map getVendorProperties(DataSource dataSource) {
        return jpaProperties.getHibernateProperties(dataSource);
    }

   /**
     * 事物管理器
     *
     * @param builder
     * @return*/

    @Bean(name = "transactionManagerPrimary")
    @Primary
    PlatformTransactionManager transactionManagerPrimary(EntityManagerFactoryBuilder builder) {
        return new JpaTransactionManager(entityManagerFactoryPrimary(builder).getObject());
    }

}

从数据库配置

@Configuration
@EnableTransactionManagement
@EnableJpaRepositories(
        entityManagerFactoryRef="entityManagerFactorySecondary",
        transactionManagerRef="transactionManagerSecondary",
        basePackages= "com.ics.top.second.repository")
public class SecondaryConfig {
    @Autowired
    private JpaProperties jpaProperties;

    @Autowired
    @Qualifier("secondaryDataSource")
    private DataSource secondaryDataSource;

    @Bean(name = "entityManagerSecondary")
    public EntityManager entityManager(EntityManagerFactoryBuilder builder) {
        return entityManagerFactorySecondary(builder).getObject().createEntityManager();
    }

    @Bean(name = "entityManagerFactorySecondary")
    public LocalContainerEntityManagerFactoryBean entityManagerFactorySecondary (EntityManagerFactoryBuilder builder) {
        return builder
            .dataSource(secondaryDataSource)
            .properties(getVendorProperties(secondaryDataSource))
            .packages("com.ics.top.second..domain") //设置实体类所在位置
            .persistenceUnit("secondaryPersistenceUnit")
            .build();
    }

    private Map getVendorProperties(DataSource dataSource) {
        return jpaProperties.getHibernateProperties(dataSource);
    }

    @Bean(name = "transactionManagerSecondary")
    PlatformTransactionManager transactionManagerSecondary(EntityManagerFactoryBuilder builder) {
        return new JpaTransactionManager(entityManagerFactorySecondary(builder).getObject());
    }

}

配置完，直接启动项目，启动成功说明好了

3.下面就是配置batch批量

 配置文件

@Configuration
@EnableBatchProcessing
@ComponentScan(basePackageClasses = DataSourceConfig.class)
public class BatchUserManagementConfiguration {
    private static final Logger log = LoggerFactory.getLogger(BatchUserManagementConfiguration.class);
    @Autowired
    public JobBuilderFactory jobBuilderFactory;
    @Autowired
    private StepBuilderFactory stepBuilderFactory;
    @Autowired
    @Qualifier("entityManagerSecondary")
    private EntityManager em;

    @Bean
    public ItemReader readerUserManagement( @Qualifier("primaryDataSource") DataSource dataSource,BatchUserManagementRowMapper userRowMapper)  {
        JdbcCursorItemReader itemReader = new JdbcCursorItemReader();
        itemReader.setDataSource(dataSource);
        String time  = DateUtil.getDatePeriod(1);
        String sql = "sql";
        log.info("查询sql语句"+sql);
        itemReader.setSql(sql);
        itemReader.setRowMapper(userRowMapper);
        itemReader.close();
        return itemReader;
    }

    @Bean
    public BatchUserManagementProcessor processorUserManagement(BatchUserManagementProcessor userItemProcessor) {
        return  userItemProcessor;
    }

    @Bean
    public ItemWriter writerUserManagement() {
        JpaItemWriter writer = new JpaItemWriter();
        try {
            writer.setEntityManagerFactory(em.getEntityManagerFactory());
            writer.afterPropertiesSet();
        } catch (Exception e) {
            e.printStackTrace();
        }

        return writer;
    }

    @Bean
    public Job userManagementJob1(JobUserManagementListener listener, Step userManagementStep1 ) {
        return jobBuilderFactory.get("userManagementJob1")//获取job
            .incrementer(new RunIdIncrementer())//
            .listener(listener)
            .start(userManagementStep1)
            .build();
    }
    @Bean
    public PlatformTransactionManager userManagementTransactionManager() {//声明事物
        return new JpaTransactionManager(em.getEntityManagerFactory());
    }

    @Bean
    public Step userManagementStep1(
        ItemReader readerUserManagement,
        ItemWriter writerUserManagement,
        BatchUserManagementProcessor processorUserManagement,
        UserManagementLoggerListener itemLoggerListener,
        @Qualifier("userManagementTransactionManager")PlatformTransactionManager userManagementTransactionManager ) {
        return stepBuilderFactory.get("userManagementStep1")
            .transactionManager(userManagementTransactionManager)//声明用那个数据源的事物
            .listener(itemLoggerListener)
            . chunk(StaticVariable.BATCH_CHUNK_COUNT)
            .reader(readerUserManagement)
            .processor(processorUserManagement)//处理数据
            .writer(writerUserManagement) //写入数据
            .build();
    }
}

这里需要注意 方法的命名，因为都声明@bean了，我们写的方法不能重复，不然后会报找不到这个bean

定时器执行任务

@Service

public class BatchUserManagementClient {

    private final Logger log = LoggerFactory.getLogger(this.getClass());
    @Autowired
    private JobLauncher jobLauncher;
    @Autowired
    @Qualifier("userManagementJob1")
    private Job job;

    @Scheduled(cron="0 30 2 * * ?") //10分钟执行一次,把拨打计划已完成的数据，同步到用户管理表去
//    @Scheduled(cron="0/10 * * * * ?") //10分钟执行一次,把拨打计划已完成的数据，同步到用户管理表去
    public void batchStart() throws Exception {
        log.info("执行了userManagementJob1定时器");
        try {
            JobParameters jobParameters =
                new JobParametersBuilder()
                    .addLong("time",System.currentTimeMillis()).toJobParameters(); // 采用这个方法，会从新执行job，添加一个新的识别长参数。
            JobExecution execution = jobLauncher.run(job, jobParameters);
            System.out.println("Execution status: "+ execution.getStatus());
        } catch (JobExecutionAlreadyRunningException | JobRestartException | JobInstanceAlreadyCompleteException
            | JobParametersInvalidException e) {
            e.printStackTrace();
        }
    }

还有些比较简单的我这里就不说了，有兴趣的可以去点击点击打开链接下载源码