SpringBoot+mybatis+druid 多数据源实现
最近搞多数据源动态切换,根据不同的场景服务切换到不动的数据源上。从而实现分库分表。
1、先看下我们项目的配置、pom.xml
4.0.0
com
boot-demo
0.0.1-SNAPSHOT
jar
boot-demo
http://maven.apache.org
org.springframework.boot
spring-boot-starter-parent
2.1.0.RELEASE
UTF-8
org.springframework.boot
spring-boot-starter-web
org.springframework.boot
spring-boot-starter-tomcat
provided
com.alibaba
druid-spring-boot-starter
1.1.10
mysql
mysql-connector-java
tk.mybatis
mapper
4.0.4
ch.qos.logback
logback-classic
tk.mybatis
mapper-spring-boot-starter
1.2.4
ch.qos.logback
logback-classic
org.springframework.boot
spring-boot-starter-logging
org.projectlombok
lombok
provided
junit
junit
3.8.1
test
org.springframework.boot
spring-boot-starter-aop
org.springframework.boot
spring-boot-starter-logging
org.springframework.boot
spring-boot-maven-plugin
application.properties
server.port=8012
server.servlet.context-path=/
server.tomcat.uri-encoding=UTF-8
# mybatis
mybatis.config-location=classpath:mybatis-config.xml
mybatis.mapper-locations=classpath:mapper/*.xml
# db 01
spring.datasource.data01.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.data01.url=jdbc:mysql://192.168.179.128:3306/TEST?characterEncoding=utf-8&useSSL=false
spring.datasource.data01.username=root
spring.datasource.data01.password=mynewPass4!
spring.datasource.data01.driverClassName=com.mysql.jdbc.Driver
spring.datasource.data01.initialSize=5
spring.datasource.data01.minIdle=5
spring.datasource.data01.maxActive=20
spring.datasource.data01.maxWait=60000
spring.datasource.data01.timeBetweenEvictionRunsMillis=60000
spring.datasource.data01.minEvictableIdleTimeMillis=300000
spring.datasource.data01.validationQuery=SELECT 1 FROM DUAL
spring.datasource.data01.testWhileIdle=true
spring.datasource.data01.testOnBorrow=false
spring.datasource.data01.testOnReturn=false
spring.datasource.data01.poolPreparedStatements=true
spring.datasource.data01.maxPoolPreparedStatementPerConnectionSize=20
#spring.datasource.data01.filters=stat,wall,log4j
spring.datasource.data01.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
# db 02
spring.datasource.data02.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.data02.url=jdbc:mysql://192.168.179.128:3306/awa?characterEncoding=utf-8&useSSL=false
spring.datasource.data02.username=root
spring.datasource.data02.password=mynewPass4!
spring.datasource.data02.driverClassName=com.mysql.jdbc.Driver
spring.datasource.data02.initialSize=5
spring.datasource.data02.minIdle=5
spring.datasource.data02.maxActive=20
spring.datasource.data02.maxWait=60000
spring.datasource.data02.timeBetweenEvictionRunsMillis=60000
spring.datasource.data02.minEvictableIdleTimeMillis=300000
spring.datasource.data02.validationQuery=SELECT 1 FROM DUAL
spring.datasource.data02.testWhileIdle=true
spring.datasource.data02.testOnBorrow=false
spring.datasource.data02.testOnReturn=false
spring.datasource.data02.poolPreparedStatements=true
spring.datasource.data02.maxPoolPreparedStatementPerConnectionSize=20
#spring.datasource.data02.filters=stat,wall,log4j
spring.datasource.data02.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
现在我们来说下实现过程；
1、编写java代码配置类
package com.boot.config;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.sql.DataSource;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
/**
* 多数据源切换,只差AOP了
*
* @author zpl
*
*/
@Configuration
public class DataSourceConfig {
@Bean(name = "dataSource01")
@ConfigurationProperties(prefix = "spring.datasource.data01") // application.properteis中对应属性的前缀
public DataSource dataSource01() {
return DruidDataSourceBuilder.create().build();
}
@Bean(name = "dataSource02")
@ConfigurationProperties(prefix = "spring.datasource.data02") // application.properteis中对应属性的前缀
public DataSource dataSource02() {
return DruidDataSourceBuilder.create().build();
}
@Bean(name = "dataSource")
@Primary // 多数据源的情况下,这个注解很关键,否则报:expected single matching bean but found 3:
// dataSource01,dataSource02,dataSou
public DataSource dataSource() {
DataSourceRouting routing = new DataSourceRouting();
DataSource dataSource01 = dataSource01();
// 设置默认数据源
routing.setDefaultTargetDataSource(dataSource01);
// 配置多数据源
Map dataSources = new ConcurrentHashMap();
dataSources.put(DataSourceRouting.DATA_SOURCE_MASTER, dataSource01);
dataSources.put(DataSourceRouting.DATA_SOURCE_SUB, dataSource02());
routing.setTargetDataSources(dataSources);
return routing;
}
// @Bean
// public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws
// Exception
// {
// SqlSessionFactoryBean sqlSessionFactoryBean = new
// SqlSessionFactoryBean();
// //设置数据源
// sqlSessionFactoryBean.setDataSource(dataSource);
// //指定基础包
// sqlSessionFactoryBean.setTypeAliasesPackage("com.microservice.dbandcache.model");
// //指定mapper位置
// sqlSessionFactoryBean.setMapperLocations(new
// PathMatchingResourcePatternResolver()
// .getResources("classpath:mapping/*.xml"));
//
// return sqlSessionFactoryBean.getObject();
// }
}
这里设置了多数据源，并且在启动类上的注解去掉自动配置数据源的配置类
@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class})
2、配置多数据源路由，需要切换数据源的时候，将threadlocal 中的副本改成需要的数据源
package com.boot.config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource;
/**
* 动态数据源路由
*
*
* @date 2018年9月21日 上午11:06:01
*
*/
public class DataSourceRouting extends AbstractRoutingDataSource {
public static final String DATA_SOURCE_MASTER = "dataSourceMaster";
public static final String DATA_SOURCE_SUB = "dataSourceSub";
private static final ThreadLocal localContext = new ThreadLocal();
private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceRouting.class);
@Override
protected Object determineCurrentLookupKey() {
if (null == localContext.get()) {
return null;
}
LOGGER.info("数据源切换至" + localContext.get());
return localContext.get();
}
public static void setMasterDataSource() {
localContext.set(DATA_SOURCE_MASTER);
}
public static void setSubDataSource() {
localContext.set(DATA_SOURCE_SUB);
}
}
3、当我们的Idao 接口类调用数据库操作时，会通过determineCurrentLookupKey获取数据源名，具体的实现原理可以看源码AbstractRoutingDataSource 的实现。
4、这样我们就可以在执行对应的方法之前先将ThreadLocal 的内容改变成需要的数据源key.从而改变数据源。
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-===============================================
再想一想，按照上面的实现感觉有点low,在每次执行方法之前就得硬编码切换一下。而我们使用的是mybatis ,我们的Idao接口都有@Mapper ，所以我们用的接口方法都是统一的。那怎么才能更便捷点呢
方法1、自定义注解，在IDao 接口上添加，并给值为指定的数据源类型。在执行方法的时候我们可以拦截到接口上的注解，从而获取到注解的数据源类型配置，从而通过路由DataSourceRouting 来这是ThreadLocal中的数据源key.
package com.boot.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import com.boot.enums.DataSourceConst;
@Target({ ElementType.TYPE, ElementType.METHOD })
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface TargetDataSource {
DataSourceConst.DataSourceType value() default DataSourceConst.DataSourceType.dataSourceMaster;
}
package com.boot.enums;
public interface DataSourceConst {
enum DataSourceType {
dataSourceMaster, dataSourceSub
}
}
package com.boot.config;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.After;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
@Component
@Aspect
@Order(-1)
public class DynamicDataSourceAspect {
@Before("execution(* com.*.mapper..*.*(..))")
public void before(JoinPoint point) {
// 本来想通过拦截接口上的注解，通过注解的值来获取需要的数据源，然后通过dataSourceRouting来设置，但是研究过后，发现无法拦截接口上的注解
}
@After("execution(* com.*.mapper..*.*(..))")
public void after(JoinPoint point) {
DataSourceRouting.clearDataSource();
}
}
DynamicDataSourceAspect 是拦截器，但是通过一天的查询测试，始终无法通过joinpoint来获取到接口上的注解，但是我们通过point.getTarget();可以看到代理mapperproxy 中的interfacemapper 中确实有该注解，却始终没有拦截到。很无语，最终查到aopUtils,网上说通过aopUtils ，springboot 2.x能拿到,但是我是始终没拿到。如果知道的请指教。。。
这种方法不行，我们只能在mybatis 中操作了
通过查询，我们可以配置@mapperscan 来给他扫描的包下的Idao类指定对应的sqlsessionFactory.
配置如下
package com.boot.mapper.first;
import org.apache.ibatis.annotations.Mapper;
import com.boot.annotation.TargetDataSource;
import com.boot.bean.User;
import com.boot.enums.DataSourceConst.DataSourceType;
@Mapper
//@TargetDataSource(DataSourceType.dataSourceMaster)
public interface IUserDao extends tk.mybatis.mapper.common.Mapper {
}
package com.boot.mapper.second;
import org.apache.ibatis.annotations.Mapper;
import com.boot.annotation.TargetDataSource;
import com.boot.bean.User;
import com.boot.enums.DataSourceConst.DataSourceType;
@Mapper
//@TargetDataSource(DataSourceType.dataSourceSub)
public interface IUserDao2 extends tk.mybatis.mapper.common.Mapper {
}
以上两类是Idao 接口的实现。继承 tk.mybatis.mapper.common.Mapper。可以实现父类接口方法来自定义方法修改实现，这个不是重点。
再来看配置类
package com.boot.config;
import javax.sql.DataSource;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import tk.mybatis.spring.annotation.MapperScan;
/**
* 分库-01库配置
*
* @author zpl
*
*/
@Configuration
@MapperScan(basePackages = "com.boot.mapper.first", sqlSessionFactoryRef = "SqlSessionFactoryfirst")
public class DataSourceFirstConfig {
@Bean(name = "dataSource01")
@ConfigurationProperties(prefix = "spring.datasource.data01") // application.properteis中对应属性的前缀
public DataSource dataSource01() {
return DruidDataSourceBuilder.create().build();
}
@Bean(name="SqlSessionFactoryfirst")
public SqlSessionFactory sqlSessionFactory(@Qualifier("dataSource01") DataSource dataSource) throws Exception {
SqlSessionFactoryBean sqlSessionFactoryBean = new SqlSessionFactoryBean();
// 设置数据源
sqlSessionFactoryBean.setDataSource(dataSource);
// 指定基础包
sqlSessionFactoryBean.setTypeAliasesPackage("com.boot.bean.*");
// 指定mapper位置
sqlSessionFactoryBean
.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath*:mapper/*.xml"));
return sqlSessionFactoryBean.getObject();
}
}
package com.boot.config;
import javax.sql.DataSource;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import tk.mybatis.spring.annotation.MapperScan;
/**
* 分库-01库配置
*
* @author zpl
*
*/
@Configuration
@MapperScan(basePackages = "com.boot.mapper.second", sqlSessionFactoryRef = "SqlSessionFactorysecond")
public class DataSourceSecondConfig {
@Bean(name = "dataSource02")
@ConfigurationProperties(prefix = "spring.datasource.data02") // application.properteis中对应属性的前缀
public DataSource dataSource02() {
return DruidDataSourceBuilder.create().build();
}
@Bean(name = "SqlSessionFactorysecond")
public SqlSessionFactory sqlSessionFactory(@Qualifier("dataSource02") DataSource dataSource) throws Exception {
SqlSessionFactoryBean sqlSessionFactoryBean = new SqlSessionFactoryBean();
// 设置数据源
sqlSessionFactoryBean.setDataSource(dataSource);
// 指定基础包
sqlSessionFactoryBean.setTypeAliasesPackage("com.boot.bean.*");
// 指定mapper位置
sqlSessionFactoryBean
.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath*:mapper/*.xml"));
return sqlSessionFactoryBean.getObject();
}
}
上面的是配置类。配置了两个数据源，并给指定的包下面Idao接口配置了指定的数据源的sqlsessionFactory.
其中遇到的问题，
1、指定mapper位置，原先写的是classpath:mapper/*.xml;
但是启动报错。错误如下：
Caused by: java.io.FileNotFoundException: class path resource [mapper/] cannot be resolved to URL because it does not exist
at org.springframework.core.io.ClassPathResource.getURL(ClassPathResource.java:195) ~[spring-core-5.1.2.RELEASE.jar:5.1.2.RELEASE]
at org.springframework.core.io.support.PathMatchingResourcePatternResolver.findPathMatchingResources(PathMatchingResourcePatternResolver.java:497) ~[spring-core-5.1.2.RELEASE.jar:5.1.2.RELEASE]
at org.springframework.core.io.support.PathMatchingResourcePatternResolver.getResources(PathMatchingResourcePatternResolver.java:298) ~[spring-core-5.1.2.RELEASE.jar:5.1.2.RELEASE]
at com.boot.config.DataSourceSecondConfig.sqlSessionFactory(DataSourceSecondConfig.java:43) ~[classes/:na]
at com.boot.config.DataSourceSecondConfig$$EnhancerBySpringCGLIB$$c22d62aa.CGLIB$sqlSessionFactory$0() ~[classes/:na]
at com.boot.config.DataSourceSecondConfig$$EnhancerBySpringCGLIB$$c22d62aa$$FastClassBySpringCGLIB$$793b95be.invoke() ~[classes/:na]
at org.springframework.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:244) ~[spring-core-5.1.2.RELEASE.jar:5.1.2.RELEASE]
at org.springframework.context.annotation.ConfigurationClassEnhancer$BeanMethodInterceptor.intercept(ConfigurationClassEnhancer.java:363) ~[spring-context-5.1.2.RELEASE.jar:5.1.2.RELEASE]
at com.boot.config.DataSourceSecondConfig$$EnhancerBySpringCGLIB$$c22d62aa.sqlSessionFactory() ~[classes/:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_181]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_181]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_181]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_181]
at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:154) ~[spring-beans-5.1.2.RELEASE.jar:5.1.2.RELEASE]
... 46 common frames omitted
找不到mapper的路径，不知道为啥，改成classpath*:mapper/*.xml 就好了。
问题2：原先@MapperScan(basePackages = "com.boot.mapper.second", sqlSessionFactoryRef = "SqlSessionFactorysecond") 中指定basePackages的值是，com.boot.mapper.second.* ,本意想让他扫com.boot.mapper.second 包中的接口类，但是启动报错，错误如下：
***************************
APPLICATION FAILED TO START
***************************
Description:
A component required a bean of type 'com.boot.mapper.second.IUserDao2' that could not be found.
Action:
Consider defining a bean of type 'com.boot.mapper.second.IUserDao2' in your configuration.
意思是创建IUserDao2 的实例bean 失败，通过尝试，分析原因是，basePackages 指定的是包名，会扫指定包下的类，如果后面多了个.* ,那么它就会扫second 包下的其他包下的接口。而自己包下的接口类就扫不到了，故创建不了bean。而我们的controller类引用了该接口实例类。所以报错。
以上问题都解决调之后，我们就尝试执行了下，如代码：
package com.boot.controller;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.Resource;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.boot.bean.User;
import com.boot.config.DataSourceRouting;
import com.boot.mapper.first.IUserDao;
import com.boot.mapper.second.IUserDao2;
@RestController
@RequestMapping("/data")
public class DataSourceController {
// @Resource
// private SqlSessionFactory sqlSessionFactory;
@Resource
private IUserDao userDao;
@Resource
private IUserDao2 userDao2;
// @RequestMapping("/swith.json")
// public void dataSwitch() {
// DataSourceRouting.setMasterDataSource();
// try {
// Connection conenction = sqlSessionFactory.openSession().getConnection();
// DataSourceRouting.setSubDataSource();
// Connection conenction2 = sqlSessionFactory.openSession().getConnection();
// System.out.println(conenction.getMetaData().getUserName());
// System.out.println(conenction.getCatalog()); // 数据库名称
// System.out.println(conenction2.getMetaData().getUserName());
// System.out.println(conenction2.getCatalog()); // 数据库名称
// } catch (SQLException e) {
// e.printStackTrace();
// }
// }
@RequestMapping("/insert.json")
public void insert() {
List list = new ArrayList();
list.add(new User(null, 1, "张三1", 1));
list.add(new User(null, 2, "张三2", 2));
list.add(new User(null, 3, "张三3", 3));
list.add(new User(null, 4, "张三4", 4));
list.add(new User(null, 5, "张三5", 5));
list.add(new User(null, 6, "张三6", 6));
for (User u : list) {
Integer userId = u.getUserId();
if (userId % 2 == 0) {
userDao.insert(u);
} else {
userDao2.insert(u);
}
}
}
}
启动的时候还报错：
***************************
APPLICATION FAILED TO START
***************************
Description:
Parameter 0 of method sqlSessionTemplate in tk.mybatis.mapper.autoconfigure.MapperAutoConfiguration required a single bean, but 2 were found:
- SqlSessionFactoryfirst: defined by method 'sqlSessionFactory' in class path resource [com/boot/config/DataSourceFirstConfig.class]
- SqlSessionFactorysecond: defined by method 'sqlSessionFactory' in class path resource [com/boot/config/DataSourceSecondConfig.class]
Action:
Consider marking one of the beans as @Primary, updating the consumer to accept multiple beans, or using @Qualifier to identify the bean that should be consumed
根据提示，我们发现，在启动springBoot的时候mapper有个自动配置，MapperAutoConfiguration，会配置sqlsessionFactory,而且只配置其中一个，导致启动失败，因为我们有两个，它不知道用哪个了。所以报错。
所以我们在启动springBoot的时候，加上
@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class,MapperAutoConfiguration.class})
不走它的自动配置，用我们自己的配置，
注意：：：
我们如果多数据源按照上面配置的话，我们的@MapperScan 注解就不能再startApplication 启动类上面加了。
有时间的话我们研究下使用tk.mybatis 实现分表。如何实现分表。以及它是如何实现查询的，，横向分表难度比较大的就是查询，连表查询，以及排序，聚合等操作查询，。主流的有mycat, sharding-sphere(原当当[sharding-jdbc]升级版本)。
thanks everyone！！！