shardingJDBC按月分表，能否实现动态创建表，然后进行分表？
如题。
现在有个任务，一张大表数据上亿，需要按照月份拆分。用shardingJDBC的技术进行分表。
这个是log数据，而且会一直增加，
也就是说，即使现在你创建了2018年12个月，2019年12个月，等到2020年一月份怎么办？
难道要一开始创建好几年的表，然后等过了这些年，再手动创建新表然后上代码改配置吗？
全网没有搜到解决方案，
到看到一个跟我一样诉求的帖子：if shardingsphere can create tables automatically? #2521 里面并没有解决办法
还有一个说是没法动态创建表：Sharding-JDBC的动态分表
但好像都没明确说明为什么不能动态分表，大概看了一下分表规则核心代码：
分表设置需要在TableRule上设置具体分表有那几个表：actualTables(tnames)
那我是不是可以在分片的时候，假如我按照goods_id%4分片，我现在已经在数据库里创建了goods_0,goods_1两张表，当goods_id%4==2或者3的时候，用JDBC创建表，然后让shardingJDBC知道我新建了一个表，分片规则里的表就多一个，不就好了嘛。
【我分析认为：
一旦创建了DataSource，就分配了分表规则，已经写死在里面了，
新建一个表，表名是不是得重新写一个分表规则，新建一个buildDataSource()?
这些都没用吧？因为人家shardingJDBC就认你一开始制定的规则，后面加的不认识。。】
如果有大佬实现了这个功能，求告诉我。。。。
现在来看看我是怎么掉坑里的，想把问题抛出来，看看有没有解决办法，
没有的话，那就只能手动建个20年的表(不知道会不会被打死)
现在来看看我的demo怎么实现的：
架构用的是springboot + Maven + JPA + MySQL +sharding-jbdc，
感兴趣的同学可以自己按照我的目录重新构建项目，代码我不放github了。
这个demo除了动态建表分表，没有成功之外， （不需要这部分代码可以自己去掉，等下告诉你删哪些代码）
它很好的实现了：（分库的部分我注释掉了，感兴趣的可以自己完善它，网上也有相关代码）
1. 按id奇偶分表；
2. 分布式自增主键；
3. 测试分表save，update，select，delete；
4. 测试查询跨表数据；
5. 测试查询指定id list数据；
6. 用java创建MySQL 数据表；
目录是： （我把无关的目录划掉了）
pom.xml :
4.0.0
org.springframework.boot
spring-boot-starter-parent
2.1.3.RELEASE
com.example.ziqiiii
sharding-jdbc-demo
0.0.1-SNAPSHOT
sharding-jdbc-demo
Demo project for Spring Boot
1.8
org.springframework.boot
spring-boot-starter-data-jpa
org.springframework.boot
spring-boot-starter-web
org.springframework.boot
spring-boot-devtools
runtime
mysql
mysql-connector-java
runtime
org.springframework.boot
spring-boot-starter-test
test
org.projectlombok
lombok
true
com.alibaba
druid
1.1.9
com.dangdang
sharding-jdbc-core
1.5.4
org.jadira.usertype
usertype.core
7.0.0.CR1
org.jadira.usertype
usertype.jodatime
2.0.1
joda-time
joda-time-hibernate
1.4
joda-time
joda-time
2.9.9
org.springframework.boot
spring-boot-maven-plugin
config下面的TableShardingAlgorithm.java
package com.example.ziqiiii.shardingjdbcdemo.config;
import com.dangdang.ddframe.rdb.sharding.api.ShardingValue;
import com.dangdang.ddframe.rdb.sharding.api.strategy.table.SingleKeyTableShardingAlgorithm;
import com.example.ziqiiii.shardingjdbcdemo.database.DataSourceConfig;
import com.google.common.collect.Range;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.sql.Connection;
import java.sql.Statement;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
/**
* create by ziqi on 2019/8/19
*/
@Component
public class TableShardingAlgorithm implements SingleKeyTableShardingAlgorithm {
private final String tablePrefix = "goods_";
@Autowired
private DataSourceConfig dataSourceConfig;
@Override
public String doEqualSharding( Collection tableNames, ShardingValue shardingValue) {
//1.
//分表：goods_0,goods_1:
// String endStr = shardingValue.getValue() % 2 + "";
// String tb = tablePrefix + endStr;
// return tb;
//下面的代码是动态创建表，并进行分表
//创建表功能OK，但是分表报错，不需要动态创建分表的话，可以注释掉下面部分
//2.
//动态创建表，假设分片规则为id%4,已经创建了0，1表，那么它会自动创建2，3表
List databsesTable = dataSourceConfig.getTnames();
String endStr = shardingValue.getValue() % 4 + "";
String tb = tablePrefix + endStr;
if(tableNames.contains(tb)){
System.out.println(tb);
}else{
String creatsql = String.format("CREATE TABLE `%s` (" +
" `goods_id` bigint(20) NOT NULL," +
" `goods_name` varchar(100) COLLATE utf8_bin NOT NULL," +
" `goods_type` bigint(20) DEFAULT NULL," +
" PRIMARY KEY (`goods_id`)" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin",tb);
Connection conn = dataSourceConfig.getConnection();
//执行创建表
System.out.println("//创建表");
try{
Statement stmt = conn.createStatement();
if(0 == stmt.executeUpdate(creatsql)) {
System.out.println("成功创建表！");
} else {
System.out.println("创建表失败！");
}
//
stmt.close();
conn.close();
System.out.println("//关闭资源");
databsesTable.add(tb);
tableNames.add(tb);
}catch (Exception e){
e.printStackTrace();
}
}
return tb;
}
@Override
public Collection doInSharding(final Collection tableNames, final ShardingValue shardingValue) {
Collection result = new LinkedHashSet<>(tableNames.size());
for (Long value : shardingValue.getValues()) {
for (String tableName : tableNames) {
if (tableName.endsWith(value % 2 + "")) {
result.add(tableName);
}
}
}
return result;
}
@Override
public Collection doBetweenSharding(final Collection tableNames,
final ShardingValue shardingValue) {
Collection result = new LinkedHashSet<>(tableNames.size());
Range range = shardingValue.getValueRange();
for (Long i = range.lowerEndpoint(); i <= range.upperEndpoint(); i++) {
for (String each : tableNames) {
if (each.endsWith(i % 2 + "")) {
result.add(each);
}
}
}
return result;
}
} controller下面的GoodsController.java
package com.example.ziqiiii.shardingjdbcdemo.controller;
import com.dangdang.ddframe.rdb.sharding.keygen.DefaultKeyGenerator;
import com.example.ziqiiii.shardingjdbcdemo.entity.Goods;
import com.example.ziqiiii.shardingjdbcdemo.repository.GoodsRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
/**
* create by ziqi on 2019/8/19
*/
@RestController
public class GoodsController {
@Autowired
private DefaultKeyGenerator keyGenerator;
@Autowired
private GoodsRepository goodsRepository;
@GetMapping("save")
public String save(){
for(int i= 11 ; i <= 20 ; i ++){
Goods goods = new Goods();
Number n = keyGenerator.generateKey();
// System.out.println("keyGenerator:"+n);
// goods.setGoodsId((long) n);
goods.setGoodsId((long) i);
goods.setGoodsName( "shi" + i);
goods.setGoodsType((long) (i+1));
goodsRepository.save(goods);
}
return "success";
}
@GetMapping("update")
public String update(){
for(int i= 1 ; i <= 10 ; i ++){
// Goods goods = new Goods();
// goods.setGoodsId((long) i);
// goods.setGoodsName( "ziqi" + i);
// goods.setGoodsType((long) (i+1));
int res = goodsRepository.update((long) i,"ziqi" + i);
}
return "success";
}
@GetMapping("select")
public String select(){
return goodsRepository.findAll().toString();
}
@GetMapping("delete")
public void delete(){
goodsRepository.deleteAll();
}
@GetMapping("query1")
public Object query1(){
return goodsRepository.findAllByGoodsIdBetween(10L, 30L);
}
@GetMapping("query2")
public Object query2(){
List goodsIds = new ArrayList<>();
goodsIds.add(10L);
goodsIds.add(15L);
goodsIds.add(20L);
goodsIds.add(25L);
return goodsRepository.findAllByGoodsIdIn(goodsIds);
}
}
database下面的Database0Config.java （这部分是分库的一个库）
package com.example.ziqiiii.shardingjdbcdemo.database;
import com.alibaba.druid.pool.DruidDataSource;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import javax.sql.DataSource;
/**
* create by ziqi on 2019/8/16
*/
@Data
@ConfigurationProperties(prefix = "database0")
@Component
public class Database0Config {
private String url;
private String username;
private String password;
private String driverClassName;
private String databaseName;
public DataSource createDataSource() {
DruidDataSource result = new DruidDataSource();
result.setDriverClassName(getDriverClassName());
result.setUrl(getUrl());
result.setUsername(getUsername());
result.setPassword(getPassword());
return result;
}
}
Database1Config.java （这部分是分库的另一个库）需要分库的这个注释可以去掉
package com.example.ziqiiii.shardingjdbcdemo.database;
/**
* create by ziqi.zhang on 2019/8/19
*/
//@Data
//@ConfigurationProperties(prefix = "database1")
//@Component
//public class Database1Config {
// private String url;
// private String username;
// private String password;
// private String driverClassName;
// private String databaseName;
//
// public DataSource createDataSource() {
// DruidDataSource result = new DruidDataSource();
// result.setDriverClassName(getDriverClassName());
// result.setUrl(getUrl());
// result.setUsername(getUsername());
// result.setPassword(getPassword());
// return result;
// }
//}DataSourceConfig.java
package com.example.ziqiiii.shardingjdbcdemo.database;
/**
* create by ziqi on 2019/8/16
*/
import com.dangdang.ddframe.rdb.sharding.api.ShardingDataSourceFactory;
import com.dangdang.ddframe.rdb.sharding.api.rule.DataSourceRule;
import com.dangdang.ddframe.rdb.sharding.api.rule.ShardingRule;
import com.dangdang.ddframe.rdb.sharding.api.rule.TableRule;
import com.dangdang.ddframe.rdb.sharding.api.strategy.table.TableShardingStrategy;
import com.dangdang.ddframe.rdb.sharding.keygen.DefaultKeyGenerator;
import com.example.ziqiiii.shardingjdbcdemo.config.TableShardingAlgorithm;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import javax.sql.DataSource;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.*;
@Configuration
public class DataSourceConfig {
private List tnames = new ArrayList<>();
//构造代码块，在构造函数调用前调用
{
tnames.add("goods_0");
tnames.add("goods_1");
}
@Autowired
private Database0Config database0Config;
@Autowired
private TableShardingAlgorithm tableShardingAlgorithm;
@Bean
public DataSource getDataSource() throws SQLException {
return buildDataSource();
}
@Bean
public List getTnames(){
return tnames;
}
public boolean setTnames(String name){
return tnames.add(name);
}
/*
* 获取sql连接
*/
public Connection getConnection() {
//原生JDBC开发
DataSource dataSource = null;
Connection conn = null;
try {
dataSource = getDataSource();
conn = dataSource.getConnection();
if (!conn.isClosed()) {
System.out.println("数据库连接成功!");
}
} catch (SQLException e) {
e.printStackTrace();
}
return conn;
}
private DataSource buildDataSource() throws SQLException {
//分库设置
Map dataSourceMap = new HashMap<>(1);
//添加两个数据库database0和database1
dataSourceMap.put(database0Config.getDatabaseName(), database0Config.createDataSource());
// dataSourceMap.put(database1Config.getDatabaseName(), database1Config.createDataSource());
//设置默认数据库
DataSourceRule dataSourceRule = new DataSourceRule(dataSourceMap, database0Config.getDatabaseName());
//分表设置，大致思想就是将查询虚拟表Goods根据一定规则映射到真实表中去
//Goods的分表规则
TableRule orderTableRule = TableRule.builder("goods")
// .actualTables(Arrays.asList("goods_0", "goods_1")) //这样写
.actualTables(tnames) //或者这样
.dataSourceRule(dataSourceRule)
.build();
//分表设置，history_按照月份划分
// List historyTables = Arrays.asList("history_0", "history_1","history_2",
// "history_3","history_4","history_5","history_6",
// "history_7","history_8","history_9","history_10",
// "history11");
//
// TableRule orderTableRule = TableRule.builder("history_wti_alarm")
// .actualTables(historyTables)
// .dataSourceRule(dataSourceRule)
// .build();
//分库分表策略
ShardingRule shardingRule = ShardingRule.builder()
.dataSourceRule(dataSourceRule)
.tableRules(Arrays.asList(orderTableRule))
.tableShardingStrategy(new TableShardingStrategy("goods_id", tableShardingAlgorithm)).build();
DataSource dataSource = ShardingDataSourceFactory.createDataSource(shardingRule);
return dataSource;
}
@Bean
public DefaultKeyGenerator keyGenerator() {
DefaultKeyGenerator kg = new DefaultKeyGenerator();
InetAddress address;
Long workerId;
try {
address = InetAddress.getLocalHost();
} catch (final UnknownHostException e) {
throw new IllegalStateException("Cannot get LocalHost InetAddress, please check your network!");
}
// 先得到服务器的hostname，例如JTCRTVDRA44，linux上可通过命令"cat /proc/sys/kernel/hostname"查看；
String hostName = address.getHostName();
try {
// 计算workerId的方式：
// 第一步hostName.replaceAll("\\d+$", "")，即去掉hostname后纯数字部分，例如JTCRTVDRA44去掉后就是JTCRTVDRA
// 第二步hostName.replace(第一步的结果, "")，即将原hostname的非数字部分去掉，得到纯数字部分，就是workerId
// workerId = Long.valueOf(hostName.replace(hostName.replaceAll("\\d+$", ""), ""));
String tmp = hostName.replaceAll("\\d+$", "");
String tmp2 = hostName.replace(tmp, "");
if(("").equals(tmp2)){
workerId = 0L;
}else {
workerId = Long.valueOf(tmp2);
}
} catch (final NumberFormatException e) {
throw new IllegalArgumentException(String.format("Wrong hostname:%s, hostname must be end with number!", hostName));
}
kg.setWorkerId(workerId);
return kg;
}
}
特地去看了一下它的.class文件，长这样：
entity下面的Goods.java
package com.example.ziqiiii.shardingjdbcdemo.entity;
import lombok.Data;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
/**
* create by ziqi on 2019/8/19
*/
@Entity
@Table(name="goods")
@Data
public class Goods {
@Id
private Long goodsId;
private String goodsName;
private Long goodsType;
}repository下面的GoodsRepository.java
package com.example.ziqiiii.shardingjdbcdemo.repository;
import com.example.ziqiiii.shardingjdbcdemo.entity.Goods;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import javax.transaction.Transactional;
import java.util.List;
/**
* create by ziqi on 2019/8/19
*/
public interface GoodsRepository extends JpaRepository {
List findAllByGoodsIdBetween(Long goodsId1, Long goodsId2);
List findAllByGoodsIdIn(List goodsIds);
@Modifying
@Transactional
@Query("update Goods g set g.goodsName =:name where g.goodsId =:id")
int update(@Param("id") Long id, @Param("name")String name);
} package com.example.ziqiiii.shardingjdbcdemo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class ShardingJdbcDemoApplication {
public static void main(String[] args) {
SpringApplication.run(ShardingJdbcDemoApplication.class, args);
}
}
resources下面的application.properties
##Jpa配置
spring.jpa.database=mysql
spring.jpa.show-sql=true
spring.jpa.hibernate.ddl-auto=none
##数据库配置
##数据库database0地址
database0.url=jdbc:mysql://localhost:3306/database0?characterEncoding=utf8&useSSL=false
##数据库database0用户名
database0.username=root
##数据库database0密码
database0.password=
##数据库database0驱动
database0.driverClassName=com.mysql.cj.jdbc.Driver
##数据库database0名称
database0.databaseName=database0
##数据库database1地址
database1.url=jdbc:mysql://localhost:3306/database1?characterEncoding=utf8&useSSL=false]
##数据库database1用户名
database1.username=root
##数据库database1密码
database1.password=
##数据库database1驱动
database1.driverClassName=com.mysql.jdbc.Driver
##数据库database1名称
database1.databaseName=database1
建表语句：
CREATE DATABASE database0;
USE database0;
DROP TABLE IF EXISTS `goods_0`;
CREATE TABLE `goods_0` (
`goods_id` bigint(20) NOT NULL,
`goods_name` varchar(100) COLLATE utf8_bin NOT NULL,
`goods_type` bigint(20) DEFAULT NULL,
PRIMARY KEY (`goods_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
DROP TABLE IF EXISTS `goods_1`;
CREATE TABLE `goods_1` (
`goods_id` bigint(20) NOT NULL,
`goods_name` varchar(100) COLLATE utf8_bin NOT NULL,
`goods_type` bigint(20) DEFAULT NULL,
PRIMARY KEY (`goods_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
CREATE DATABASE database1;
USE database1;
DROP TABLE IF EXISTS `goods_0`;
CREATE TABLE `goods_0` (
`goods_id` bigint(20) NOT NULL,
`goods_name` varchar(100) COLLATE utf8_bin NOT NULL,
`goods_type` bigint(20) DEFAULT NULL,
PRIMARY KEY (`goods_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
DROP TABLE IF EXISTS `goods_1`;
CREATE TABLE `goods_1` (
`goods_id` bigint(20) NOT NULL,
`goods_name` varchar(100) COLLATE utf8_bin NOT NULL,
`goods_type` bigint(20) DEFAULT NULL,
PRIMARY KEY (`goods_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;