springboot实现读写分离
读写分离/多数据源配置
技术选型
- springboot
- mybatis
- mysql
实现关键点
使用springboot实现mysql的读写分离，或者说多数据源配置，最关键的一点就是实现：sql的动态路由
即对于一个要执行的sql，系统自动判断这个sql将要在哪个mysql服务器上执行，也就是需要实现数据源的动态切换。
spring提供了一个用于切换数据源的抽象类：
org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource
可以集成这个抽象类，通过实现其determineCurrentLookupKey方法来返回一个 可用的数据源唯一标识，即数据源的key,以下是该抽象类的源码：
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by Fernflower decompiler)
//
package org.springframework.jdbc.datasource.lookup;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import java.util.function.BiConsumer;
import javax.sql.DataSource;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.jdbc.datasource.AbstractDataSource;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
public abstract class AbstractRoutingDataSource extends AbstractDataSource implements InitializingBean {
@Nullable
private Map targetDataSources;
@Nullable
private Object defaultTargetDataSource;
private boolean lenientFallback = true;
private DataSourceLookup dataSourceLookup = new JndiDataSourceLookup();
@Nullable
private Map resolvedDataSources;
@Nullable
private DataSource resolvedDefaultDataSource;
public AbstractRoutingDataSource() {
}
public void setTargetDataSources(Map targetDataSources) {
this.targetDataSources = targetDataSources;
}
public void setDefaultTargetDataSource(Object defaultTargetDataSource) {
this.defaultTargetDataSource = defaultTargetDataSource;
}
public void setLenientFallback(boolean lenientFallback) {
this.lenientFallback = lenientFallback;
}
public void setDataSourceLookup(@Nullable DataSourceLookup dataSourceLookup) {
this.dataSourceLookup = (DataSourceLookup)(dataSourceLookup != null ? dataSourceLookup : new JndiDataSourceLookup());
}
public void afterPropertiesSet() {
if (this.targetDataSources == null) {
throw new IllegalArgumentException("Property 'targetDataSources' is required");
} else {
this.resolvedDataSources = new HashMap(this.targetDataSources.size());
this.targetDataSources.forEach((key, value) -> {
Object lookupKey = this.resolveSpecifiedLookupKey(key);
DataSource dataSource = this.resolveSpecifiedDataSource(value);
this.resolvedDataSources.put(lookupKey, dataSource);
});
if (this.defaultTargetDataSource != null) {
this.resolvedDefaultDataSource = this.resolveSpecifiedDataSource(this.defaultTargetDataSource);
}
}
}
protected Object resolveSpecifiedLookupKey(Object lookupKey) {
return lookupKey;
}
protected DataSource resolveSpecifiedDataSource(Object dataSource) throws IllegalArgumentException {
if (dataSource instanceof DataSource) {
return (DataSource)dataSource;
} else if (dataSource instanceof String) {
return this.dataSourceLookup.getDataSource((String)dataSource);
} else {
throw new IllegalArgumentException("Illegal data source value - only [javax.sql.DataSource] and String supported: " + dataSource);
}
}
public Connection getConnection() throws SQLException {
return this.determineTargetDataSource().getConnection();
}
public Connection getConnection(String username, String password) throws SQLException {
return this.determineTargetDataSource().getConnection(username, password);
}
public T unwrap(Class iface) throws SQLException {
return iface.isInstance(this) ? this : this.determineTargetDataSource().unwrap(iface);
}
public boolean isWrapperFor(Class iface) throws SQLException {
return iface.isInstance(this) || this.determineTargetDataSource().isWrapperFor(iface);
}
protected DataSource determineTargetDataSource() {
Assert.notNull(this.resolvedDataSources, "DataSource router not initialized");
Object lookupKey = this.determineCurrentLookupKey();
DataSource dataSource = (DataSource)this.resolvedDataSources.get(lookupKey);
if (dataSource == null && (this.lenientFallback || lookupKey == null)) {
dataSource = this.resolvedDefaultDataSource;
}
if (dataSource == null) {
throw new IllegalStateException("Cannot determine target DataSource for lookup key [" + lookupKey + "]");
} else {
return dataSource;
}
}
@Nullable
protected abstract Object determineCurrentLookupKey();
}
其中：determineTargetDataSource方法是真正返回数据源的方法,内部执行了以下一段代码：
Object lookupKey = this.determineCurrentLookupKey();
DataSource dataSource = (DataSource)this.resolvedDataSources.get(lookupKey);
if (dataSource == null && (this.lenientFallback || lookupKey == null)) {
dataSource = this.resolvedDefaultDataSource;
}
调用了determineCurrentLookupKey方法来获取数据源的key，从resolvedDataSources中拿到真正的数据源
而resolvedDataSources的赋值来自哪里呢？继续看源码发现以下这个方法中操作了赋值：
public void afterPropertiesSet() {
if (this.targetDataSources == null) {
throw new IllegalArgumentException("Property 'targetDataSources' is required");
} else {
this.resolvedDataSources = new HashMap(this.targetDataSources.size());
this.targetDataSources.forEach((key, value) -> {
Object lookupKey = this.resolveSpecifiedLookupKey(key);
DataSource dataSource = this.resolveSpecifiedDataSource(value);
this.resolvedDataSources.put(lookupKey, dataSource);
});
if (this.defaultTargetDataSource != null) {
this.resolvedDefaultDataSource = this.resolveSpecifiedDataSource(this.defaultTargetDataSource);
}
}
}
该方法顾名思义：就是在类参数得到set后执行的，使用一个循环来将targetDataSources的内容放到`resolvedDataSources中，以及默认的数据源resolvedDefaultDataSource来源于defaultTargetDataSource，到这里就很明了，如果我们要实现自定义的多数据源切换，重点就是：
- 实现AbstractRoutingDataSource
- 实现获取数据源的key 的方法determineCurrentLookupKey
- 将所有数据源封装成map赋值给targetDataSources
- 设置一个默认数据源赋值给defaultTargetDataSource；
代码如下
新建一个实现类
public class MyRoutingDataSource extends AbstractRoutingDataSource {
@Override
protected Object determineCurrentLookupKey() {
return DBContextHolder.get();
}
}
DBContextHolder持有一个数据源的key，通过get方法来获取
@Slf4j
public class DBContextHolder {
private static final ThreadLocal contextHolder = new ThreadLocal<>();
public static void set(DBTypeEnum dbTypeEnum) {
contextHolder.set(dbTypeEnum);
}
public static DBTypeEnum get(){
return contextHolder.get();
}
public static void master() {
set(DBTypeEnum.MASTER);
log.info("切换到主库");
}
public static void slave() {
set(DBTypeEnum.SLAVE);
log.info("切换到从库");
}
}
其中DBTypeEnum是一个标志不同数据源的枚举
public enum DBTypeEnum {
MASTER,SLAVE;
}
在DBContextHolder中使用了ThreadLocal来存储数据源的key，是为了保证并发情况下每一个线程都有自己独立的数据源的key，互不干扰,同时提供了master(),salve()两个方法用于数据源标识的切换
接下来是最重要的一部分：数据源的配置
@Configuration
public class DataSourceConfig {
@Bean
@ConfigurationProperties("spring.datasource.master")
public DataSource masterDataSource() {
return DataSourceBuilder.create().build();
}
@Bean
@ConfigurationProperties("spring.datasource.slave")
public DataSource slaveDataSource() {
return DataSourceBuilder.create().build();
}
@Bean
public DataSource myRoutingDataSource(@Qualifier("masterDataSource") DataSource masterDataSource,
@Qualifier("slaveDataSource") DataSource slaveDataSource) {
Map dataSourceMap = new HashMap<>();
dataSourceMap.put(DBTypeEnum.MASTER,masterDataSource);
dataSourceMap.put(DBTypeEnum.SLAVE,slaveDataSource);
MyRoutingDataSource myRoutingDataSource = new MyRoutingDataSource();
myRoutingDataSource.setTargetDataSources(dataSourceMap);
myRoutingDataSource.setDefaultTargetDataSource(masterDataSource);
return myRoutingDataSource;
}
}
这里就是通过两个set方法实现了多个数据源的注入以及默认数据源的注入
如果要实现读写分离，可以使用aop进行拦截，将要执行的sql分为两类，一类是修改数据库的，一类是读取数据库的，对不同的分组执行相应的数据源切换操作
@Aspect
@Component
public class DataSourceAop {
@Pointcut("!@annotation(com.cb.separation.annotation.Master) " +
"&& (execution(* com.cb.separation.service..*.select*(..)) " +
"|| execution(* com.cb.separation.service..*.get*(..))" +
"|| execution(* com.cb.separation.service..*.find*(..))" +
"|| execution(* com.cb.separation.service..*.query*(..)))")
public void slavePointcut() {
}
/*主库的切点,或者标注了Master注解或者方法名为insert、update等开头的方法，走主库*/
@Pointcut("@annotation(com.cb.separation.annotation.Master) " +
"|| execution(* com.cb.separation.service..*.insert*(..)) " +
"|| execution(* com.cb.separation.service..*.add*(..)) " +
"|| execution(* com.cb.separation.service..*.update*(..)) " +
"|| execution(* com.cb.separation.service..*.edit*(..)) " +
"|| execution(* com.cb.separation.service..*.delete*(..)) " +
"|| execution(* com.cb.separation.service..*.remove*(..))")
public void masterPointcut() {
}
@Before("slavePointcut()")
public void slave() {
DBContextHolder.slave();
}
@Before("masterPointcut()")
public void master() {
DBContextHolder.master();
}
}
由于读写分离的mysql架构，会存在一定的时效问题，即从库的数据可能比主库的数据要晚一些达到同步，所以对于一些时效性要求特别高的sql，可以继续在主库上执行，实现方式就是增加一个注解@Master,需要在主库上执行的方法，在其上面加上该注解，aop对其进行拦截即可
数据库配置：
spring:
datasource:
master:
jdbc-url: jdbc:mysql://localhost:3306/order_master?serverTimezone=GMT%2b8&autoReconnect=true&useUnicode=true&characterEncoding=utf-8&autocommit=false
username: root
password: root
driver-class-name: com.mysql.cj.jdbc.Driver
slave:
jdbc-url: jdbc:mysql://localhost:3306/order_slave?serverTimezone=GMT%2b8&autoReconnect=true&useUnicode=true&characterEncoding=utf-8&autocommit=false
username: root
password: root
driver-class-name: com.mysql.cj.jdbc.Driver
logging:
level:
cn.enjoyedu.rwseparation: DEBUG
root: INFO
发现的问题：
对于带有@Transactional的方法，其内部的查询方法，虽然得到了拦截，但是读sql的执行还是在主库上进行
@Transactional
public void insertOrders(int orderNumber){
Random r = new Random();
OrderExp orderExp ;
for(int i=0;i
从库中是空表，这里读取sql的执行结果不应该是30，而应该是0，说明是在主库上进行的
当把事务注解去掉：
发现数据源进行了成功的切换而且执行也是在从库中
由此联想到，@Transactional注解标识的方法在执行完毕前始终使用的是同一个数据库连接，直到方法执行完毕。可以使用编程式事务来代替该注解
@Autowired
private TransactionTemplate transactionTemplate;
public void insertOrders(final int orderNumber){
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus transactionStatus) {
Random r = new Random();
OrderExp orderExp ;
int num = orderNumber;
try {
for (int i = 0; i < num; i++) {
long expireTime = r.nextInt(20) + 5;//订单的超时时长，单位秒5~25
orderExp = new OrderExp();
String orderNo = "DD00_" + expireTime + "S";//订单的编号
orderExp.setOrderNo(orderNo);
orderExp.setOrderNote("海王5排" + expireTime + "号,过期时长:" + orderNo);
orderExp.setOrderStatus(UNPAY);//订单的状态，目前为未支付
orderExpMapper.insertDelayOrder(orderExp, expireTime);
log.info("保存订单到DB:" + orderNo);
if (i == 3) {
throw new RuntimeException("ceshi");
}
}
} catch (Exception e) {
e.printStackTrace();
transactionStatus.setRollbackOnly();
}
}
});
//查询
this.findOrders();
}
这样需要事务的那部分代码并不会一直占用数据库连接，从而可以达到目的
完整代码地址：
https://github.com/kkll1314/mysql-rw-seperation