spring4开发SpringBatch 样例 -配置文件版
本文根据《SpringBatch批处理框架》一书作者刘相的源码升级spring4.0，源码为3.2版本
经测试可用
提示：本文源码比较多
遇到的问题总结在后面
配置
数据库初始化 create-tables-mysql.sql
DROP TABLE IF EXISTS t_credit;
DROP TABLE IF EXISTS t_destcredit;
CREATE TABLE t_credit
(ID VARCHAR(64),
ACCOUNTID VARCHAR(20),
NAME VARCHAR(10),
AMOUNT NUMERIC(10,2),
DATE VARCHAR(20),
ADDRESS VARCHAR(128),
FLAG VARCHAR(10),
primary key (ID)
)
ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE t_destcredit
(ID VARCHAR(64),
ACCOUNTID VARCHAR(20),
NAME VARCHAR(10),
AMOUNT NUMERIC(10,2),
DATE VARCHAR(20),
ADDRESS VARCHAR(128),
primary key (ID)
)
ENGINE=InnoDB DEFAULT CHARSET=utf8;
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('01','4047390012345678','tom',100.00,'2013-2-2 12:00:08','Lu Jia Zui road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('02','4047390012345678','tom',320.00,'2013-2-3 10:35:21','Lu Jia Zui road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('03','4047390012345678','tom',674.70,'2013-2-6 16:26:49','South Linyi road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('04','4047390012345678','tom',793.20,'2013-2-9 15:15:37','Longyang road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('05','4047390012345678','tom',360.00,'2013-2-11 11:12:38','Longyang road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('06','4047390012345678','tom',100.00,'2013-2-2 12:00:08','Lu Jia Zui road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('07','4047390012345678','tom',320.00,'2013-2-3 10:35:21','Lu Jia Zui road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('08','4047390012345678','tom',674.70,'2013-2-6 16:26:49','South Linyi road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('09','4047390012345678','tom',793.20,'2013-2-9 15:15:37','Longyang road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('10','4047390012345678','tom',360.00,'2013-2-11 11:12:38','Longyang road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('11','4047390012345678','tom',100.00,'2013-2-2 12:00:08','Lu Jia Zui road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('12','4047390012345678','tom',320.00,'2013-2-3 10:35:21','Lu Jia Zui road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('13','4047390012345678','tom',674.70,'2013-2-6 16:26:49','South Linyi road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('14','4047390012345678','tom',793.20,'2013-2-9 15:15:37','Longyang road','false');
INSERT INTO t_credit(ID,ACCOUNTID,NAME,AMOUNT,DATE,ADDRESS,FLAG) VALUES('15','4047390012345678','tom',360.00,'2013-2-11 11:12:38','Longyang road','false');
数据库连接配置 batch-mysql.properties
datasource.driver=com.mysql.jdbc.Driver
datasource.url=jdbc:mysql://127.0.0.1:3306/test1289
datasource.username=root
datasource.password=000000
Spring DB驱动配置 job-context.xml
Bean定义 job-context.xml
${datasource.driver}
${datasource.url}
Batch的分区任务配置 job-partition-db.xml
源码
数据源PoJo-CreditBill.java
/**
*
*/
package com.demo.report.settle;
import java.io.Serializable;
/**
* 信用卡对账单模型.
*
* @author bruce.liu(mailto:[email protected])
* 2014-1-11下午02:29:58
*/
public class CreditBill implements Serializable{
/**
*
*/
private static final long serialVersionUID = -4940717436114184875L;
private String id;
private String accountID = ""; /** 银行卡账户ID */
private String name = ""; /** 持卡人姓名 */
private double amount = 0; /** 消费金额 */
private String date; /** 消费日期 ，格式YYYY-MM-DD HH:MM:SS*/
private String address; /** 消费场所 **/
public CreditBill(){}
public CreditBill(String accountID, String name, double amount, String date, String address){
this.accountID = accountID;
this.name = name;
this.amount = amount;
this.date = date;
this.address = address;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getAccountID() {
return accountID;
}
public void setAccountID(String accountID) {
this.accountID = accountID;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public double getAmount() {
return amount;
}
public void setAmount(double amount) {
this.amount = amount;
}
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
/**
*
*/
public String toString(){
StringBuffer sb = new StringBuffer();
sb.append("id=" + getId() + ";accountID=" + getAccountID() + ";name=" + getName() + ";amount="
+ getAmount() + ";date=" + getDate() + ";address=" + getAddress());
return sb.toString();
}
}
接收方PoJo DestinationCreditBill.java
/**
*
*/
package com.demo.report.settle;
import java.io.Serializable;
/**
*
* @author bruce.liu(mailto:[email protected])
* 2014-1-11下午02:37:55
*/
public class DestinationCreditBill implements Serializable{
/**
*
*/
private static final long serialVersionUID = 5253572139260172440L;
private String id;
private String accountID = ""; /** 银行卡账户ID */
private String name = ""; /** 持卡人姓名 */
private double amount = 0; /** 消费金额 */
private String date; /** 消费日期 ，格式YYYY-MM-DD HH:MM:SS*/
private String address; /** 消费场所 **/
public DestinationCreditBill(){}
public DestinationCreditBill(String accountID, String name, double amount, String date, String address){
this.accountID = accountID;
this.name = name;
this.amount = amount;
this.date = date;
this.address = address;
}
public String getAccountID() {
return accountID;
}
public void setAccountID(String accountID) {
this.accountID = accountID;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public double getAmount() {
return amount;
}
public void setAmount(double amount) {
this.amount = amount;
}
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
/**
*
*/
public String toString(){
StringBuffer sb = new StringBuffer();
sb.append("accountID=" + getAccountID() + ";name=" + getName() + ";amount="
+ getAmount() + ";date=" + getDate() + ";address=" + getAddress());
return sb.toString();
}
}
分区步骤监视器 PartitionStepExecutionListener.java
/**
*
*/
package com.demo.report.settle;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.StepExecutionListener;
/**
*
* @author bruce.liu(mailto:[email protected])
* 2014-3-22上午10:11:29
*/
public class PartitionStepExecutionListener implements StepExecutionListener {
@Override
public void beforeStep(StepExecution stepExecution) {
System.out.println("ThreadName=" + Thread.currentThread().getName() + "; "
+ "StepName=" + stepExecution.getStepName() + ";");
}
@Override
public ExitStatus afterStep(StepExecution stepExecution) {
return null;
}
}
查询结果集转换器 CreditBillRowMapper.java
/**
*
*/
package com.demo.report.settle;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.springframework.jdbc.core.RowMapper;
import com.demo.report.settle.CreditBill;
/**
*
* @author bruce.liu(mailto:[email protected])
* 2014-1-11下午03:10:28
*/
public class CreditBillRowMapper implements RowMapper {
public CreditBill mapRow(ResultSet rs, int rowNum) throws SQLException {
CreditBill bill = new CreditBill();
bill.setId(rs.getString("DID"));
bill.setAccountID(rs.getString("ACCOUNTID"));
bill.setAddress(rs.getString("ADDRESS"));
bill.setAmount(rs.getDouble("AMOUNT"));
bill.setDate(rs.getString("INDATE"));
bill.setName(rs.getString("INNAME"));
return bill;
}
}
分区策略生成器 DBpartition.java
package com.demo.report.settle;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.Map;
import javax.sql.DataSource;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.jdbc.core.JdbcTemplate;
/**
*
* @author bruce.liu(mailto:[email protected])
* 2014-1-11下午03:13:45
*/
public class DBpartition implements Partitioner {
private static final String _MINRECORD = "_minRecord";
private static final String _MAXRECORD = "_maxRecord";
private static final String _GRIDSIZE = "_grideSize";
private static final String _CURPARTITION = "_curPartition";
private static final String MIN_SELECT_PATTERN = "select min({0}) from {1}";
private static final String MAX_SELECT_PATTERN = "select max({0}) from {1}";
private JdbcTemplate jdbcTemplate ;
private DataSource dataSource;
private String table ;
private String column;
public Map partition(int gridSize) {
validateAndInit();
Map resultMap = new HashMap();
int min = jdbcTemplate.queryForObject(MessageFormat.format(MIN_SELECT_PATTERN, new Object[]{column,table}),Integer.class);
int max = jdbcTemplate.queryForObject(MessageFormat.format(MAX_SELECT_PATTERN, new Object[]{column,table}),Integer.class);
int targetSize = (max-min)/gridSize +1;
int number=0;
int start =min;
int end = start+targetSize-1;
while(start <= max){
ExecutionContext context = new ExecutionContext();
if(end>=max){
end=max;
}
context.putInt(_MINRECORD, start);
context.putInt(_MAXRECORD, end);
context.putLong(_CURPARTITION, Thread.currentThread().getId());
context.putInt(_GRIDSIZE, gridSize);
start+=targetSize;
end+=targetSize;
resultMap.put("partition"+(number++), context);
}
System.out.println("gridSize is ="+gridSize+"***current thread id is "+Thread.currentThread().getId()+"*** name is "+Thread.currentThread().getName());
if(!resultMap.isEmpty())
{
resultMap.forEach((k,v)->{System.out.println("key is "+k+"****values is"+v);});
}
return resultMap;
}
public void validateAndInit(){
if(isEmpty(table)){
throw new IllegalArgumentException("table cannot be null");
}
if(isEmpty(column)){
throw new IllegalArgumentException("column cannot be null");
}
if(dataSource!=null && jdbcTemplate==null){
jdbcTemplate = new JdbcTemplate(dataSource);
}
if(jdbcTemplate==null){
throw new IllegalArgumentException("jdbcTemplate cannot be null");
}
}
public static boolean isEmpty(String info){
if(info!=null){
if(info.trim().length()>1){
return false;
}
}
return true;
}
public String getColumn() {
return column;
}
public void setColumn(String column) {
this.column = column;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public JdbcTemplate getJdbcTemplate() {
return jdbcTemplate;
}
public void setJdbcTemplate(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
public DataSource getDataSource() {
return dataSource;
}
public void setDataSource(DataSource dataSource) {
this.dataSource = dataSource;
}
}
分区处理器 CreditBillProcessor.java
/**
*
*/
package com.demo.report.settle;
import org.springframework.batch.item.ItemProcessor;
/**
*
* @author bruce.liu(mailto:[email protected])
* 2014-1-11下午02:38:01
*/
public class CreditBillProcessor implements
ItemProcessor {
public DestinationCreditBill process(CreditBill bill) throws Exception {
System.out.println(bill.toString());
DestinationCreditBill destCreditBill = new DestinationCreditBill();
destCreditBill.setAccountID(bill.getAccountID());
destCreditBill.setAddress(bill.getAddress());
destCreditBill.setAmount(bill.getAmount());
destCreditBill.setDate(bill.getDate());
destCreditBill.setId(bill.getId());
destCreditBill.setName(bill.getName());
return destCreditBill;
}
}
测试类基类 JobLaunchBase.java
package com.snail.report.settle;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
/**
*
*/
/**
* @author bruce.liu(mailto:[email protected])
* 2013-9-7下午01:57:02
*/
public class JobLaunchBase {
private static ApplicationContext context;
/**
* 执行批处理作业.
* @param jobPath 作业配置文件
* @param jobName 作业名
* @param builder 作业参数构造器
*/
public static void executeJob(String jobPath, String jobName, JobParametersBuilder builder) {
context = new ClassPathXmlApplicationContext(jobPath);
JobLauncher launcher = (JobLauncher) context.getBean("jobLauncher");
Job job = (Job) context.getBean(jobName);
try {
JobExecution result = launcher.run(job, builder.toJobParameters());
System.out.println(result.toString());
} catch (Exception e) {
e.printStackTrace();
}finally {
context=null;
job=null;
builder=null;
launcher=null;
}
}
}
测试类 JobLaunchPartitionDBOracle.java
package com.snail.report.settle;
/**
*
*/
import java.util.Date;
import org.springframework.batch.core.JobParametersBuilder;
import com.snail.report.settle.JobLaunchBase;
/**
*
* @author bruce.liu(mailto:[email protected])
* 2013-11-16下午10:59:46
*/
public class JobLaunchPartitionDBOracle {
/**
* @param args
*/
public static void main(String[] args) {
JobLaunchBase.executeJob("db/job-partition-db.xml", "partitionJob",
new JobParametersBuilder().addDate("date", new Date()));
System.exit(0);
}
}
问题总结
找不到batch
原因是创建spring boot项目时忘记加batch了
3.2.1.RELEASE
org.springframework.batch
spring-batch-core
${spring.batch.version}
jar
false
org.springframework.batch
spring-batch-infrastructure
${spring.batch.version}
jar
false
org.springframework.batch
spring-batch-integration
1.2.0.RELEASE
false
找不到oracle驱动
手工加入maven仓库
mvn install:install-file -DgroupId=com.oracle -DartifactId=ojdbc14 -Dversion=10.2.0.1.0 -Dpackaging=jar -Dfile=ojdbc14-10.2.0.1.0.jar
无法连续事务处理
ORA-08177: 无法连续访问此事务处理 batch
org.springframework.context.support.AbstractRefreshableApplicationContext.refreshBeanFactory(AbstractRefreshableApplicationContext.java:131)
at org.springframework.context.support.AbstractApplicationContext.obtainFreshBeanFactory(AbstractApplicationContext.java:537)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:451)
at org.springframework.context.support.ClassPathXmlApplicationContext.(ClassPathXmlApplicationContext.java:139)
at org.springframework.context.support.ClassPathXmlApplicationContext.(ClassPathXmlApplicationContext.java:83)
解决办法 job-repository 改事务级别
jdbcTemplate找不到 queryForInt 方法
The method queryForInt(String) is undefined for the type JdbcTemplate DBpartition.java
因为使用spring4，默认jdbc版本也高了，原有的方法没了，所以解决办法是改成 jdbcTemplate.queryForObject 注意要新加一个参数是 返回对象的类型
int max = jdbcTemplate.queryForObject(MessageFormat.format(MAX_SELECT_PATTERN, new Object[]{column,table}),**Integer.class**);
加了mod，细化分区
加这个算是一点试验，刚开始没注意到变量名的问题，执行错误，后来发现
配置文件中的where条件如果取变量值 ，需要stepExecutionContext+变量名 才可以
参考我加的**_grideSize**就明白了。
总结
总体来说改造难度不大，下一次版本应该是注解版的了，或许挑战一下分区多线程。
推荐文章
http://www.kailing.pub/article/index/arcid/196.html
https://www.cnblogs.com/softidea/p/7142084.html