springboot+mybatis+sharding-jdbc单库分表

缘由

由于在做考勤系统考虑到后期考勤数据量会非常大，到时候单表的数据量可能会很大，导致数据库的查询效率会很低，数据库的负载坚持不住，所以就开始考虑分库分表的事情了，一开始是看的比较主流的当初阿里图队的mycat，但是整合到后面发现mycat实现不了单库分表，只能多库分表。后面就开始往sharding-jdbc方向研究，sharding-jdbc又分为两个版本，dangdang（当当）和shardingsphere，也不知道这两者是什么关系应该都是同个社区的开发者开发的吧，当当的目前最新是1.5.3吧shardingsphere最近发布了3.1.0版本，越来越接近原生的jdbc支持，但是由于最新版本集成maven依赖报错所以我的项目用的是3.0.0.M3版本

sharding-jdbc》当当 sql限制

不支持DDL语句

不支持子语句

不支持UNION 和 UNION ALL

不支持特殊INSERT

每条INSERT语句只能插入一条数据，不支持VALUES后有多行数据的语句 就是批零操作

不支持DISTINCT聚合

不支持存储过程

不支持or关键字查询

shardingsphere》sql限制

有所差别的是shardingsphere

支持批量操作

支持or 关键字查询

其它的基本和当当的限制一致

具体选哪个版本看你们的业务需求了

正文配置

springboot版本2.1

 

pom.xml

        
		
		
	      io.shardingsphere
	      sharding-jdbc
	      3.0.0.M3
    	

application.properties配置

# \u6570\u636E\u6E90\u914D\u7F6E
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
#spring.datasource.url=jdbc:mysql://localhost:3306/face_recog?characterEncoding\=utf-8&zeroDateTimeBehavior\=convertToNull&transformedBitIsBoolean\=true
spring.datasource.jdbc-url=jdbc:mysql://localhost:3306/face_recog?useUnicode=true&zeroDateTimeBehavior=convertToNull&useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT%2B8
spring.datasource.username=root
spring.datasource.password=root
spring.datasource.name=face_recog
# \u8FDE\u63A5\u6C60\u914D\u7F6E
# \u521D\u59CB\u5316\u5927\u5C0F\uFF0C\u6700\u5C0F\uFF0C\u6700\u5927
spring.datasource.initialSize=1
spring.datasource.minIdle=1
spring.datasource.maxActive=20

数据源配置

package com.faceRecog.manage.sharding;

 


import com.faceRecog.manage.strategy.ModuloDatabaseShardingAlgorithm;
import com.faceRecog.manage.strategy.ModuloTableShardingAlgorithm;
import com.zaxxer.hikari.HikariDataSource;

import io.shardingsphere.core.api.ShardingDataSourceFactory;
import io.shardingsphere.core.api.config.ShardingRuleConfiguration;
import io.shardingsphere.core.api.config.TableRuleConfiguration;
import io.shardingsphere.core.api.config.strategy.StandardShardingStrategyConfiguration;

import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.SqlSessionTemplate;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;

import javax.sql.DataSource;

import java.sql.SQLException;
import java.util.*;

/**
 * 
 * @ClassName: DataSourceConfig 
 * @Description: 数据源及分表配置
 * @author: xya
 * @date: 2019年7月6日 上午10:54:01
 */
@Configuration
@MapperScan(basePackages = "com.faceRecog.manage.mapper", sqlSessionTemplateRef  = "shardSqlSessionTemplate")
public class DataSourceConfig {

		@Value("${spring.datasource.name}")
	    private String databaseName;// 数据库名
	
	 	@ConfigurationProperties(prefix = "spring.datasource")
	    @Bean(name = "dataSource1")
	    public DataSource dataSource1() {
	        return   new HikariDataSource();
	    }


	    @Primary
	    @Bean(name = "shardingDataSource")
	    public DataSource getDataSource(@Qualifier("dataSource1") DataSource dataSource1) throws SQLException {
	        ShardingRuleConfiguration shardingRuleConfig = new ShardingRuleConfiguration();
	        
	        shardingRuleConfig.getTableRuleConfigs().add(getOrderTableRuleConfiguration());// 添加数据库分表配置
	        shardingRuleConfig.getTableRuleConfigs().add(getOrigSignTableRuleConfiguration());
	        shardingRuleConfig.getBindingTableGroups().add("fr_attend_detail,fr_original_sign_record");// 需要分表的表的集合
	        Map dataSourceMap = new HashMap<>();
	        dataSourceMap.put(databaseName, dataSource1);// 数据源
	        Properties properties = new Properties();
	        
	        return ShardingDataSourceFactory.createDataSource(dataSourceMap, shardingRuleConfig, new HashMap(),properties);
	    }
	    
	    /**
	     * 
	    * @Title: getOrderTableRuleConfiguration 
	    * @Description: 第一张表的分表配置 
	    * @return TableRuleConfiguration
	    * @author xya
	    * @date 2019年7月6日上午11:01:30
	     */
	    private static TableRuleConfiguration getOrderTableRuleConfiguration() {
	        TableRuleConfiguration result = new TableRuleConfiguration();
	        result.setLogicTable("attend_detail");//逻辑表的名称就是mybatis xml中使用的持久数据库的中转名 attend_detail指向fr_attend_detail_2019和2020两张表
	        result.setActualDataNodes("face_recog.fr_attend_detail_${2019..2020}");//分表的数据库的表名 需要现在数据库建立两张表
	        //表的分片策略即根据那个数据库表字段来分配 这里使用的是create_time 还有使用的是自定义的分表策略类 ModuloDatabaseShardingAlgorithm.class
	        result.setDatabaseShardingStrategyConfig(new StandardShardingStrategyConfiguration("create_time",  new ModuloDatabaseShardingAlgorithm()));
	        result.setTableShardingStrategyConfig( new StandardShardingStrategyConfiguration("create_time",  new ModuloTableShardingAlgorithm()));
	        return result;
	    }
	    /**
	     * 
	    * @Title: getOrigSignTableRuleConfiguration 
	    * @Description: 第二张表的分表配置 
	    * @return TableRuleConfiguration
	    * @author xya
	    * @date 2019年7月6日上午11:01:34
	     */
	    private static TableRuleConfiguration getOrigSignTableRuleConfiguration() {
	        TableRuleConfiguration result = new TableRuleConfiguration();
	        result.setLogicTable("original_sign_record");
	        result.setActualDataNodes("face_recog.fr_original_sign_record_${2019..2020}");
	        result.setDatabaseShardingStrategyConfig(new StandardShardingStrategyConfiguration("create_time",  new ModuloDatabaseShardingAlgorithm()));
	        result.setTableShardingStrategyConfig( new StandardShardingStrategyConfiguration("create_time",  new ModuloTableShardingAlgorithm()));
	        return result;
	    }
	    
	    
	    @Bean(name = "shardSqlSessionFactory")
	    public SqlSessionFactory shardSqlSessionFactory(@Qualifier("shardingDataSource") DataSource dataSource) throws Exception {
	        SqlSessionFactoryBean bean = new SqlSessionFactoryBean();
	        bean.setDataSource(dataSource);
	        bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath:mapper/*.xml"));
	        //bean.setConfigLocation(new DefaultResourceLoader().getResource("classpath:mybatis/mybatis-config.xml"));
	        return bean.getObject();
	    }

	    @Bean(name = "shardTransactionManager")
	    public DataSourceTransactionManager shardTransactionManager(@Qualifier("shardingDataSource") DataSource dataSource) {
	        return new DataSourceTransactionManager(dataSource);
	    }

	    @Bean(name = "shardSqlSessionTemplate")
	    public SqlSessionTemplate shardSqlSessionTemplate(@Qualifier("shardSqlSessionFactory") SqlSessionFactory sqlSessionFactory) throws Exception {
	        return new SqlSessionTemplate(sqlSessionFactory);
	    }
}

分库策略

/**   
 * Copyright © 2019 All rights reserved.
 * 
 * @Package: com.faceRecog.manage.strategy 
 * @author: xya
 * @date: 2019年6月18日 下午5:16:33 
 */
package com.faceRecog.manage.strategy;


import com.faceRecog.manage.util.fileUpload.CommonUtil;

import io.shardingsphere.core.api.algorithm.sharding.PreciseShardingValue;
import io.shardingsphere.core.api.algorithm.sharding.standard.PreciseShardingAlgorithm;

import java.io.InputStream;
import java.util.Collection;
import java.util.Map;
 
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 分库策略的简单实现
 * Created by Kane on 2018/1/22.
 */
public   class ModuloDatabaseShardingAlgorithm implements PreciseShardingAlgorithm {
	private static Logger log=LoggerFactory.getLogger(ModuloDatabaseShardingAlgorithm.class);
	
	InputStream ins = this.getClass().getClassLoader().getResourceAsStream("application.properties");
    Map map = CommonUtil.readSafePro(ins);
    private String databaseName=map.get("spring.datasource.name");
	
    
    @Override
    public String doSharding(final Collection availableTargetNames, final PreciseShardingValue shardingValue) {
        int size = availableTargetNames.size();
        for (String each : availableTargetNames) {
            if (each.endsWith(databaseName)) {
            	return each;
            }
        }
        throw new UnsupportedOperationException();
    }
	
}

分表策略

/**   
 * Copyright © 2019  All rights reserved.
 * 
 * @Package: com.faceRecog.manage.strategy 
 * @author: xya
 * @date: 2019年6月18日 下午5:12:26 
 */
package com.faceRecog.manage.strategy;




import io.shardingsphere.core.api.algorithm.sharding.PreciseShardingValue;
import io.shardingsphere.core.api.algorithm.sharding.standard.PreciseShardingAlgorithm;

import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;


import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** 
 * @ClassName: ModuloTableShardingAlgorithm 
 * @Description: TODO
 * @author: xya
 * @date: 2019年6月18日 下午5:12:26  
 */
/**
 * 分表策略的基本实现
 * Created by Kane on 2018/1/22.
 */
public class ModuloTableShardingAlgorithm implements PreciseShardingAlgorithm {

	private static Logger log=LoggerFactory.getLogger(ModuloTableShardingAlgorithm.class);
	
	
	@Override
    public String doSharding(final Collection availableTargetNames, final PreciseShardingValue shardingValue) {
        int size = availableTargetNames.size();
        for (String each : availableTargetNames) {
        	//组装多个分片键的值，确定其对应的目标数据表
            SimpleDateFormat fmt= new SimpleDateFormat("yyyy");
            String year="";
			year = fmt.format(shardingValue.getValue());
			//return each+"_"+year;// 生成分配的的数据表表名
			if (each.endsWith(year)) {
				return each;
			}
        }
        throw new UnsupportedOperationException();
    }
	
   /* @Override
    public String doEqualSharding(Collection tableNames, ShardingValue shardingValue) {
    	log.info("[数据库策略] 当前SQL语句为：=条件语句， 分片建的值为：{}", new GsonBuilder().create().toJson(shardingValue));
        for (String each : tableNames) {
        	//组装多个分片键的值，确定其对应的目标数据表
            SimpleDateFormat fmt= new SimpleDateFormat("yyyy");
            String year="";
			year = fmt.format(shardingValue.getValue());
			//return each+"_"+year;// 生成分配的的数据表表名
			if (each.endsWith(year)) {
				return each;
			}
        }
        throw new IllegalArgumentException();
    }



	 (non Javadoc) 
	 * @Title: doInSharding
	 * @Description: TODO
	 * @param availableTargetNames
	 * @param shardingValue
	 * @return 
	 * @see com.dangdang.ddframe.rdb.sharding.router.strategy.SingleKeyShardingAlgorithm#doInSharding(java.util.Collection, com.dangdang.ddframe.rdb.sharding.api.ShardingValue) 
	  
	@Override
	public Collection doInSharding(Collection availableTargetNames,
			ShardingValue shardingValue) {
		// TODO Auto-generated method stub
		return null;
	}



	 (non Javadoc) 
	 * @Title: doBetweenSharding
	 * @Description: TODO
	 * @param availableTargetNames
	 * @param shardingValue
	 * @return 
	 * @see com.dangdang.ddframe.rdb.sharding.router.strategy.SingleKeyShardingAlgorithm#doBetweenSharding(java.util.Collection, com.dangdang.ddframe.rdb.sharding.api.ShardingValue) 
	  
	@Override
	public Collection doBetweenSharding(Collection availableTargetNames,
			ShardingValue shardingValue) {
		// TODO Auto-generated method stub
		return null;
	}

    @Override
    public Collection doInSharding(Collection tableNames, ShardingValue shardingValue) {
        Collection result = new LinkedHashSet<>(tableNames.size());
        for (String each : tableNames) {
        	//组装多个分片键的值，确定其对应的目标数据表
            SimpleDateFormat fmt= new SimpleDateFormat("yyyy");
            String year="";
			try {
				year = fmt.format(fmt.parse(shardingValue.getValue()));
				if (each.endsWith(year)) {
					return each;
				}
			} catch (ParseException e) {
				e.printStackTrace();
			}
        }
        return result;
    }

    @Override
    public Collection doBetweenSharding(Collection tableNames, ShardingValue shardingValue) {
        Collection result = new LinkedHashSet<>(tableNames.size());
        Range range = (Range) shardingValue.getValueRange();
        for (Long i = range.lowerEndpoint(); i <= range.upperEndpoint(); i++) {
            for (String each : tableNames) {
                if (each.endsWith(i % 2 + "")) {
                    result.add(each);
                }
            }
        }
        return result;
    }*/
}

到这里就算配置完了手工！！！