ClickHouse（十七）轮询写本地表之多分片动态数据源

前言

笔者操作的clickhouse集群，配置是24台机、12分片、2副本的，由于写入大量是数据到ck分布式表会导致zk压力过大，所以采用轮询写入本地表的方式，通过map存在每个分片的数据源链接来确定mybatis使用的数据源，达到写入不同本地表的效果。

1.项目文件目录：

2.数据源配置，通过查询system库得到每个分片信息

    
        select cluster,shard_num,host_name,host_address from system.clusters where replica_num=1 and cluster='ck_cluster'
        order by shard_num
    

import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
import com.test.config.dynamic.DynamicDataSource;
import com.test.dao.clickhouse.CKClusterMapper;
import com.test.domain.entity.CKNode;
import com.test.domain.entity.CkCluster;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

@Configuration
@MapperScan(basePackages = "com.test.dao.dynamic.clickhouse", sqlSessionFactoryRef = "ckDynamicSqlSessionFactory")
public class ClickHouseDynamicDataSourceConfig {

    @Autowired
    private CKClusterMapper ckClusterMapper; // 获取ck集群节点的mapper

    @Value("${clickhouse.conf.nodePort}")
    private int ckNodePort;
    @Value("${spring.datasource.clickhouse.driver-class-name}")
    private String ckDriver;
    @Value("${spring.datasource.clickhouse.username}")
    private String ckUserName;
    @Value("${spring.datasource.clickhouse.password}")
    private String ckPassword;


    @Bean
    @Qualifier("ckCluster")
    public CkCluster ckCluster() {
        List ckNodes = ckClusterMapper.selectClusterNode();
        CkCluster ckCluster = new CkCluster();
        ckCluster.setCkNodes(ckNodes);
        return ckCluster;
    }

    @Bean
    @Qualifier("ckDynamicDataSource")
    public DynamicDataSource ckDynamicDataSource(@Qualifier("ckCluster") CkCluster ckCluster){
        DynamicDataSource dynamicDataSource = new DynamicDataSource();

        // 初始化数据连接池
        Map dataSourceMap = new HashMap<>();
        List ckNodes = ckCluster.getCkNodes();
        for (int i = 0; i < ckNodes.size(); i++) {
            CKNode ckNode = ckNodes.get(i);

            DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create();
            dataSourceBuilder.driverClassName(ckDriver);
            dataSourceBuilder.url("jdbc:clickhouse://" + ckNode.getHostAddress()+":"+ckNodePort + "/default?socket_timeout=1800000");
            dataSourceBuilder.username(ckUserName);
            dataSourceBuilder.password(ckPassword);

            dataSourceMap.put(String.valueOf(i), dataSourceBuilder.build());
        }

        dynamicDataSource.setTargetDataSources(dataSourceMap);
        dynamicDataSource.setDefaultTargetDataSource(dataSourceMap.get("0"));

        return dynamicDataSource;
    }

    @Bean
    @Qualifier("ckDynamicSqlSessionFactory")
    public SqlSessionFactory ckDynamicSqlSessionFactory(@Qualifier("ckDynamicDataSource") DynamicDataSource ckDynamicDataSource) throws Exception {
        final MybatisSqlSessionFactoryBean sessionFactory = new MybatisSqlSessionFactoryBean();
        sessionFactory.setConfigLocation(new ClassPathResource("config/mybatis-config.xml"));
        sessionFactory.setDataSource(ckDynamicDataSource);
        return sessionFactory.getObject();
    }


}

3.动态DataSource设置类

@Log
public class DynamicDataSource extends AbstractRoutingDataSource {

    /**
     * 获取当前数据源对应的key
     * @return
     */
    @Override
    protected Object determineCurrentLookupKey() {
        log.info("Current DataSource is: "+DynamicDataSourceContextHolder.getDataSourceKey());
        return DynamicDataSourceContextHolder.getDataSourceKey();
    }

    @Override
    public void setTargetDataSources(Map targetDataSources) {
        super.setTargetDataSources(targetDataSources);
        // 刷新所有的数据源
        super.afterPropertiesSet();
    }

}

4.数据源使用类

public class DynamicDataSourceContextHolder {

    private static final ThreadLocal contextHolder = new ThreadLocal<>();

    /**
     * 获取当前的Data Source key
     *
     * @return
     */
    public static String getDataSourceKey() {
        return contextHolder.get();
    }

    /**
     * 设置DataSource key
     *
     * @param key
     */
    public static synchronized void setDataSourceKey(String key) {
        contextHolder.set(key);
    }

    /**
     * 清理DataSource key
     */
    public static void clearDataSourceKey() {
        contextHolder.remove();
    }

}

5.应用：

//list数据事先进行keyby操作，key为0、1、、、11对应不同数据源。以此确定mybatis mapper使用的数据源，轮询写入不同本地表
for (Map.Entry> logList : logListMap.entrySet()) {
      String shardNo = logList.getKey();
      DynamicDataSourceContextHolder.setDataSourceKey(shardNo);
      if (logList.getValue().size()>0){
          log.info("list编号："+shardNo+"，list大小："+logList.getValue().size());
          insertlocalCK(logList.getValue());
          logList.getValue().clear();
      }
  }