spring Batch实现数据库大数据量读写

spring Batch实现数据库大数据量读写

1. data-source-context.xml

2. quart-zcontext.xml commit-interval="10000"每次批量数据的条数，数值越大效率越高，

可在此处添加事物处理，每次回滚数就是commit-interval数

  
  
  
      
      
      
      
          
      
      
      
      
          
      
  
      
      
          
              
          
       
          
          
            
                
            
             
        
          
       
  
   
  
  
      
       
      
  
   
  
  
  
    
    
        
            
            
        
        
            
        execute    
        
    
   
    
        
        
            
        
        
        
        
          
        0 45 10 * * ? *    
        
         
    
       
    
    
        
        
            
                
                
            
        
    
    
  /beans>    

3.定时调度job类 QuartzLedgerJob.java 

package net.etongbao.vasp.ac.quartz;

import java.util.Date;  
import org.slf4j.Logger;  
import org.slf4j.LoggerFactory;  
import org.springframework.batch.core.Job;  
import org.springframework.batch.core.JobParametersBuilder;  
import org.springframework.batch.core.launch.JobLauncher;  
import org.springframework.beans.factory.annotation.Autowired;  
import org.springframework.stereotype.Component;  
import org.springframework.util.StopWatch;  
  
/** 
 * 定时调度类 
 * @author Fu Wei 
 * 
 */  
  
@Component("quartzLedgerJob")  
public class QuartzLedgerJob {  
  
    private static final Logger LOG = LoggerFactory.getLogger(QuartzLedgerJob.class);  
  
    @Autowired  
    private JobLauncher jobLauncher;  
  
    @Autowired  
    private Job ledgerJob;  
  
    @Autowired  
    JobParametersBuilder jobParameterBulider;  
  
    private static long counter = 0l;  
      
    /** 
     * 执行业务方法 
     * @throws Exception 
     */  
    public void execute() throws Exception {  
        LOG.debug("start...");  
        StopWatch sw = new StopWatch();  
        sw.start();  
        /* 
         * Spring Batch Job同一个job instance，成功执行后是不允许重新执行的【失败后是否允许重跑， 
         * 可通过配置Job的restartable参数来控制，默认是true】，如果需要重新执行，可以变通处理， 
         * 添加一个JobParameters构建类,以当前时间作为参数，保证其他参数相同的情况下却是不同的job instance 
         */  
        jobParameterBulider.addDate("date", new Date());  
        jobLauncher.run(ledgerJob, jobParameterBulider.toJobParameters());  
        sw.stop();  
        LOG.debug("Time elapsed:{},Execute quartz ledgerJob:{}", sw.prettyPrint(), ++counter);  
    }  
}  

4.程序启动类 StartQuartz.java 

package net.etongbao.vasp.ac.quartz;

import java.io.FileNotFoundException;  
  
import org.springframework.context.support.ClassPathXmlApplicationContext;  
  
/** 
 * 启动定时调度 
 * @author Fu Wei 
 * 
 */  
public class StartQuartz {  
  
    public static void main(String[] args) throws FileNotFoundException {  
  
        new ClassPathXmlApplicationContext("/net/etongbao/vasp/ac/resources/quartz-context.xml");  
    }  
}  

5. pojo类  Ledger.java

package net.etongbao.vasp.ac.pojo;  
  
import java.io.Serializable;  
import java.util.Date;  
  
public class Ledger implements Serializable {  
  
    private int id;  
    private Date receiptDate;  
    private String memberName;  
    private String checkNumber;  
    private Date checkDate;  
    private String paymentType;  
    private double depositAmount;  
    private double paymentAmount;  
    private String comments;  
  
    public Ledger() {  
        super();  
    }  
  
    public Ledger(int id, Date receiptDate, String memberName, String checkNumber, Date checkDate, String paymentType,  
            double depositAmount, double paymentAmount, String comments) {  
        super();  
        this.id = id;  
        this.receiptDate = receiptDate;  
        this.memberName = memberName;  
        this.checkNumber = checkNumber;  
        this.checkDate = checkDate;  
        this.paymentType = paymentType;  
        this.depositAmount = depositAmount;  
        this.paymentAmount = paymentAmount;  
        this.comments = comments;  
    }  
    public int getId() {  
        return id;  
    }  
  
    public void setId(int id) {  
        this.id = id;  
    }  
  
    public Date getReceiptDate() {  
        return receiptDate;  
    }  
  
    public void setReceiptDate(Date receiptDate) {  
        this.receiptDate = receiptDate;  
    }  
  
    public String getMemberName() {  
        return memberName;  
    }  
  
    public void setMemberName(String memberName) {  
        this.memberName = memberName;  
    }  
  
    public String getCheckNumber() {  
        return checkNumber;  
    }  
  
    public void setCheckNumber(String checkNumber) {  
        this.checkNumber = checkNumber;  
    }  
  
    public Date getCheckDate() {  
        return checkDate;  
    }  
  
    public void setCheckDate(Date checkDate) {  
        this.checkDate = checkDate;  
    }  
    public String getPaymentType() {  
        return paymentType;  
    }  
  
    public void setPaymentType(String paymentType) {  
        this.paymentType = paymentType;  
    }  
  
    public double getDepositAmount() {  
        return depositAmount;  
    }  
  
    public void setDepositAmount(double depositAmount) {  
        this.depositAmount = depositAmount;  
    }  
  
    public double getPaymentAmount() {  
        return paymentAmount;  
    }  
  
    public void setPaymentAmount(double paymentAmount) {  
        this.paymentAmount = paymentAmount;  
    }  
  
    public String getComments() {  
        return comments;  
    }  
  
    public void setComments(String comments) {  
        this.comments = comments;  
    }  
}  

6. LedgerDaoImpl.java

    package net.etongbao.vasp.ac.dao.impl;

import java.sql.PreparedStatement;  
import java.sql.SQLException;  
import net.etongbao.vasp.ac.dao.LedgerDao;  
import net.etongbao.vasp.ac.pojo.Ledger;  
import org.springframework.beans.factory.annotation.Autowired;  
import org.springframework.jdbc.core.JdbcTemplate;  
import org.springframework.jdbc.core.PreparedStatementSetter;  
import org.springframework.stereotype.Repository;  
  
/** 
 * ledger数据操作类 
 *  
 * @author Fu Wei 
 *  
 */  
  
@Repository  
public class LedgerDaoImpl implements LedgerDao {  
  
    private static final String SAVE_SQL = "insert into ledger_temp (rcv_dt, mbr_nm, chk_nbr, chk_dt, pymt_typ, dpst_amt, pymt_amt, comments) values(?,?,?,?,?,?,?,?)";  
  
    @Autowired  
    private JdbcTemplate jdbcTemplate;  
  
    @Override  
    public void save(final Ledger item) {  
        jdbcTemplate.update(SAVE_SQL, new PreparedStatementSetter() {  
            public void setValues(PreparedStatement stmt) throws SQLException {  
                stmt.setDate(1, new java.sql.Date(item.getReceiptDate().getTime()));  
                stmt.setString(2, item.getMemberName());  
                stmt.setString(3, item.getCheckNumber());  
                stmt.setDate(4, new java.sql.Date(item.getCheckDate().getTime()));  
                stmt.setString(5, item.getPaymentType());  
                stmt.setDouble(6, item.getDepositAmount());  
                stmt.setDouble(7, item.getPaymentAmount()); 
                stmt.setString(8, item.getComments());  
            }  
        });  
    }   
}  

7. 接口 LedgerDao .java

package net.etongbao.vasp.ac.dao;  
  
import net.etongbao.vasp.ac.pojo.Ledger;  
  
public interface LedgerDao {  
    public void save(final Ledger item) ;  
}  

 8. JdbcTemplete 需要的LedgerRowMapper.java

     package net.etongbao.vasp.ac.batch.writer;

import java.sql.ResultSet;  
import java.sql.SQLException;  
  
import net.etongbao.vasp.ac.pojo.Ledger;  
  
import org.springframework.jdbc.core.RowMapper;  
import org.springframework.stereotype.Component;  
   
/** 
 * ledger行的映射类 
 * @author Administrator 
 * 
 */  
@Component("ledgerRowMapper")  
public class LedgerRowMapper implements RowMapper {  
    public Object mapRow(ResultSet rs, int rowNum) throws SQLException {  
        Ledger ledger = new Ledger();  
        ledger.setId(rs.getInt("id"));  
        ledger.setReceiptDate(rs.getDate("rcv_dt"));  
        ledger.setMemberName(rs.getString("mbr_nm"));  
        ledger.setCheckNumber(rs.getString("chk_nbr"));  
        ledger.setCheckDate(rs.getDate("chk_dt"));  
        ledger.setPaymentType(rs.getString("pymt_typ"));  
        ledger.setDepositAmount(rs.getDouble("dpst_amt"));  
        ledger.setPaymentAmount(rs.getDouble("pymt_amt"));  
        ledger.setComments(rs.getString("comments"));  
        return ledger;  
    }  
}  

9.关键类LedgerWriter.java，写入数据，负责数据的添加 

package net.etongbao.vasp.ac.batch.writer;  
  
import java.util.List;  
  
import net.etongbao.vasp.ac.dao.LedgerDao;  
import net.etongbao.vasp.ac.pojo.Ledger;  
  
import org.springframework.batch.item.ItemWriter;  
import org.springframework.beans.factory.annotation.Autowired;  
import org.springframework.stereotype.Component;  
  
/** 
 * ledger写入数据 
 *  
 * @author Fu Wei 
 *  
 */  
@Component("ledgerWriter")  
public class LedgerWriter implements ItemWriter {  
  
    @Autowired  
    private LedgerDao ledgerDao;  
  
    /** 
     * 写入数据 
     *  
     * @param ledgers 
     */  
    public void write(List ledgers) throws Exception {  
        for (Ledger ledger : ledgers) {  
            ledgerDao.save(ledger);  
        }  
    }    
}  

classPath:

 总结: 测试数据8万多条，响应时间3分多钟。

 

关键在于quartz-context.xml中

class="org.springframework.batch.item.database.JdbcCursorItemReader">

 

   

 

 

负责读取数据在程序执行时一次性抓取全部数据后在批量的交给LedgerWriter进行写操作。当然也可以使用分页读取JdbcPagingItemReader，但要分页数量与写入数量要大写相

同，还可以对分页出来的数据进行添加悲观锁LedgerWriter.java 负责写入数据，每次写入1000条。