springbatch和定时器读取txt文件批量导入数据库
百度云案例下载地址：
spring batch批量读取txt文件demo案例下载：链接：https://pan.baidu.com/s/1gfRP0cF 密码：uxni
启动：找到SpringbatchApplication，右键--run as --java application（springboot启动方式）
目录结构：
1、springBatch的job的step方法有reader，processor，writer方法。
reader从本地文件读取内容，processor对读取的每行数据进行处理的过程，writer，将处理过的数据进行存储操作。
（1）第一种启动springbatch创建job方式
尽量不要用@Autowired(required = false)
package com.feeling.mc.batch.control;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.feeling.mc.common.utils.DateUtil;
import com.feeling.mc.db.entity.BatchManager;
import com.feeling.mc.db.mapper.messager.BatchManagerMapper;
@Component
public class DemoController implements ApplicationContextAware{
@Autowired(required = false)
JobLauncher jobLauncher;
private ApplicationContext applicationContext;
public JobParameters jobParameters;
@Autowired
private BatchManagerMapper managerMapper;
//@Scheduled(cron = "0 0 8 * * ?") // 早上8点
// 每分钟跑一次
@Scheduled(cron = "0 0/1 * * * ?")
public void imp() throws Exception {
Job job = (Job)this.applicationContext.getBean("ReaderBatchMessage");
jobParameters = new JobParametersBuilder().addLong("time", System.currentTimeMillis()).toJobParameters();
jobLauncher.run(job, jobParameters);
}
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
}
（2）第二种启动springbatch创建job方式
在SpringbatchApplication加上@EnableScheduling//定时器
package com.feeling.batch.controller;
import javax.annotation.Resource;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Component
public class BatchController {
@Autowired
JobLauncher jobLauncher;
@Autowired
JobOperator jobOperator;
@Resource(name="messagebatchinsertjob")
private Job batchJob;
/**
* 每天读取txt文件，
* 并且把txt文件数据处理后保存到新的txt中
* 代表一个简单的界面来启动Job一个给定的一组 JobParameters
* JobLauncher.run(Job job, JobParameters jobParameters)
* @throws Exception
*/
@Scheduled(cron = "0 0/1 * * * ?")
public void job3() throws Exception {
JobExecution run = jobLauncher.run(batchJob, newJobParametersBuilder().addLong("time",System.currentTimeMillis()).toJobParameters());
run.getId();
}
}
package com.feeling.batch.job;
import java.io.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.batch.item.file.transform.FieldSet;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.validation.BindException;
import com.feeling.batch.bean.UserEntity;
import com.feeling.batch.exception.BatchStepExceptionHandler;
import com.feeling.batch.listener.BatchJobListener;
import com.feeling.batch.proccess.BatchItemProcessor;
import com.feeling.batch.util.DateUtil;
import com.feeling.batch.writer.BatchItemWriter;
@Configuration
public class BatchJob {
private static final Logger logger = LoggerFactory.getLogger(BatchJob.class);
@Autowired
public JobBuilderFactory jobBuilderFactory;
@Autowired
public StepBuilderFactory stepBuilderFactory;
@Autowired
public PlatformTransactionManager platformTransactionManager;
@Autowired
public BatchStepExceptionHandler exceptionHandler;
@Autowired
public BatchItemWriter batchitemwriter;
@Autowired
public BatchItemProcessor batchitemprocessor;
/**
* 构建job
* 创建bean，然后用@Resource(name="batchJob")创建对象
* 1、当第二天重启前一天的任务时！！！文件日期有异
* @param listener
* @return
*/
@Bean("messagebatchinsertjob")
public Job MessageBatchInsertJob(BatchJobListener listener) {
return jobBuilderFactory.get("MessageBatchInsertJob").listener(listener).flow(MessageBatchInsertStep()).end()
.build();
}
/**
* 声明发送到MQ step
* 1、Skip:如果处理过程中某条记录是错误的,如CSV文件中格式不正确的行,那么可以直接跳过该对象,继续处理下一个。
* 2、在chunk元素上定义skip-limit属性,告诉Spring最多允许跳过多少个items,超过则job失败
* 3、Restart:如果将job状态存储在数据库中,而一旦它执行失败, 那么就可以选择重启job实例, 并继续上次的执行位置。
* 4、最后,对于执行失败的job作业,我们可以重新启动,并让他们从上次断开的地方继续执行。要达到这一点,只需要使用和上次 一模一样的参数来启动job,
* 则Spring Batch会自动从数据库中找到这个实例然后继续执行。你也可以拒绝重启,或者参数控 制某个 job中的一个tep可以重启的次数(一般来说多次重试都失败了,那我们可能需要放弃。)
*
* @return
*/
@Bean
public Step MessageBatchInsertStep() {
logger.info("MessageBatchInsertStep");
return stepBuilderFactory.get("MessageBatchInsertStep").chunk(100).reader(fileRead()).processor(batchitemprocessor)
.writer(batchitemwriter).faultTolerant().skip(Exception.class).skipLimit(100)
.taskExecutor(new SimpleAsyncTaskExecutor()).startLimit(10).allowStartIfComplete(true)
.exceptionHandler(exceptionHandler) // 设置并发方式执行exceptionHandler,异常时打印日志并抛出异常
.throttleLimit(10) // 并发任务数为 10,默认为4
.transactionManager(platformTransactionManager).build();
}
public FlatFileItemReader fileRead() {
System.out.println("fileRead()方法开始");
FlatFileItemReader fileRead = new FlatFileItemReader<>();
fileRead.setEncoding("UTF-8");
fileRead.setResource(new FileSystemResource(new File("E:\\user.txt")));
//fileRead.setLinesToSkip(2);跳过开头多少行
DefaultLineMapper lineMapper = new DefaultLineMapper();
lineMapper.setLineTokenizer(new DelimitedLineTokenizer(","));
lineMapper.setFieldSetMapper(new FieldSetMapper() {
@Override
public UserEntity mapFieldSet(FieldSet fieldSet) throws BindException {
UserEntity user = new UserEntity();
try {
user.setUsername(fieldSet.readString(0));
user.setAge(fieldSet.readInt(1));
user.setSex(fieldSet.readChar(2));
user.setBirthday(DateUtil.parseDate(fieldSet.readString(3)));
} catch (Exception e) {
logger.error("解析异常："+e.getMessage());
}
return user;
}
});
fileRead.setLineMapper(lineMapper);
return fileRead;
}
}
3、proccesser类
package com.feeling.batch.proccess;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.stereotype.Component;
import com.feeling.batch.bean.UserEntity;
@Component
public class BatchItemProcessor implements ItemProcessor {
@Override
public UserEntity process(UserEntity user) throws Exception {
// TODO Auto-generated method stub
return user;
}
}
4、writer类
package com.feeling.mc.batch.writer;
import java.util.List;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.feeling.mc.batch.service.McMessageService;
import com.feeling.mc.core.module.McMessage;
@Component
@StepScope
public class ReaderMessageBatchWriter implements ItemWriter{
@Autowired
private McMessageService mcMessageService;
@Override
public void write(List items) throws Exception {
try {
for (McMessage msg : items) {//在此可以进行对数据存储到数据库操作
mcMessageService.addMessageBatch(msg);
}
} catch (Exception e) {
}
}
}
5、application.properties
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.url=jdbc:oracle:thin:@127.0.0.1:1521:orcl
spring.datasource.driver-class-name=oracle.jdbc.driver.OracleDriver
spring.datasource.username=admin
spring.datasource.password=admin
#启动时不启动job
spring.batch.job.enabled=false
#初始化批量
spring.batch.initializer.enabled=true6、springboot方式启动类
package com.feeling.batch;
import javax.sql.DataSource;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.transaction.PlatformTransactionManager;
@Configuration
@ComponentScan(basePackages = { "com.feeling.batch.*" }) // 将该包下的文件纳入容器中
@EnableAutoConfiguration
@EnableBatchProcessing//springbatch
@EnableScheduling//定时器
@MapperScan(basePackages = { "com.feeling.batch.mapper" })//mybatis的Mapper层扫描
public class SpringbatchApplication {
public static void main(String[] args) {
SpringApplication.run(SpringbatchApplication.class, args);
}
/**
* JobRepository是上述所有定型的持久机制。它提供了CRUD操作JobLauncher，Job以及 Step实现。
* 当 Job第一次启动，一个 JobExecution被从库中获得，和执行过程中StepExecution和 JobExecution实现方式是通过将它们传递到存储库坚持：
* @param dataSource
* @param transactionManager
* @return
*/
@Bean
public JobRepository jobRepositoryFactoryBean(DataSource dataSource,PlatformTransactionManager transactionManager){
JobRepositoryFactoryBean jobRepositoryFactoryBean = new JobRepositoryFactoryBean();
jobRepositoryFactoryBean.setTransactionManager(transactionManager);
jobRepositoryFactoryBean.setDataSource(dataSource);
jobRepositoryFactoryBean.setIsolationLevelForCreate("ISOLATION_READ_COMMITTED");
try {
jobRepositoryFactoryBean.afterPropertiesSet();
return jobRepositoryFactoryBean.getObject();
} catch (Exception e) {
//logger.error("创建jobRepositoryFactoryBean异常：{}",e.getMessage());
}
return null;
}
}
package com.feeling.batch.listener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobExecutionListener;
import org.springframework.stereotype.Component;
@Component
public class BatchJobListener implements JobExecutionListener {
private static final Logger log = LoggerFactory.getLogger(BatchJobListener.class);
public void afterJob(JobExecution jobExecution) {
log.info("任务处理结束");
}
public void beforeJob(JobExecution jobExecution) {
log.info("任务处理开始");
}
}8、springBatch的异常处理类
package com.feeling.batch.exception;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.JobInterruptedException;
import org.springframework.batch.repeat.RepeatContext;
import org.springframework.batch.repeat.exception.DefaultExceptionHandler;
import org.springframework.stereotype.Component;
@Component
public class BatchStepExceptionHandler extends DefaultExceptionHandler {
private static final Logger logger = LoggerFactory.getLogger(BatchStepExceptionHandler.class);
@Override
public void handleException(RepeatContext context, Throwable throwable) throws Throwable {
logger.error("Step运行时异常："+throwable.getMessage());
throw new JobInterruptedException("Step运行时异常："+throwable.getMessage());
}
}
springbatch，数据库mybatis，密码加密，spring类型，jdk，tomcat，定时器等jar包的引入
4.0.0
com.feeling.batch
springbatchdemo
0.0.1-SNAPSHOT
UTF-8
4.3.9.RELEASE
org.springframework.boot
spring-boot-starter-parent
1.4.5.RELEASE
com.alibaba
druid-spring-boot-starter
1.1.6
org.springframework.boot
spring-boot-starter-test
test
com.feeling
oracle.ojdbc
11.2.0
org.springframework.boot
spring-boot-starter-web
org.mybatis.spring.boot
mybatis-spring-boot-starter
1.1.1
org.springframework
spring-context
org.springframework
spring-core
org.springframework
spring-beans
org.springframework
spring-tx
org.springframework
spring-context-support
org.springframework
spring-webmvc
org.springframework
spring-jms
org.springframework
spring-orm
org.springframework.batch
spring-batch-core
org.springframework.boot
spring-boot-starter-batch
org.springframework
spring-core
junit
junit
test
org.apache.maven.plugins
maven-compiler-plugin
2.5.1
1.7
1.7
org.apache.maven.plugins
maven-jar-plugin
false
true
lib/
com.feeling.mc.admin.AdminAppliction
org.apache.maven.plugins
maven-dependency-plugin
copy
package
copy-dependencies
${project.build.directory}/lib
src/main/webapp
META-INF/resources
**/**
src/main/resources
**/**
false
package com.feeling.batch.bean;
import java.util.Date;
public class UserEntity {
private String username;
private int age;
private char sex;
private Date birthday;
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public int getAge() {
return age;
}
public void setAge(int age) {
this.age = age;
}
public char getSex() {
return sex;
}
public void setSex(char sex) {
this.sex = sex;
}
public Date getBirthday() {
return birthday;
}
public void setBirthday(Date birthday) {
this.birthday = birthday;
}
public UserEntity(String username, int age, char sex, Date birthday) {
super();
this.username = username;
this.age = age;
this.sex = sex;
this.birthday = birthday;
}
public UserEntity() {
super();
// TODO Auto-generated constructor stub
}
}