springboot+mybatis整合clickhouse实现批量插入

一、clickhouse表的创建

客户端连接工具建议使用：DBeaver

CREATE TABLE IF NOT EXISTS temp.xxxx_user_behaviors_url_log (
 `transfer_date` Date,
 `record_time` String,
 `user` String,
 `group` String,
 `host_ip` String,
 `dst_ip` String,
 `serv` String,
 `app` String,
 `site` String,
 `tm_type` String,
 `net_action` String,
 `url` String,
 `dns` String,
 `title` String,
 `snapshot` String
) ENGINE = MergeTree() PARTITION BY toYYYYMMDD(transfer_date) ORDER BY transfer_date SETTINGS index_granularity = 8192

二、mybatis、clickhouse相关maven依赖

<dependency>
	 <groupId>ru.yandex.clickhousegroupId>
	 <artifactId>clickhouse-jdbcartifactId>
	 <version>0.2.4version>
dependency>

<dependency>
   <groupId>org.mybatis.spring.bootgroupId>
   <artifactId>mybatis-spring-boot-starterartifactId>
   <version>2.1.4version>
dependency>
        
<dependency>
     <groupId>com.alibabagroupId>
     <artifactId>druid-spring-boot-starterartifactId>
     <version>1.1.21version>
dependency>

---

三、配置clickhouse数据源

spring:
  datasource:
    type: com.alibaba.druid.pool.DruidDataSource
    url: jdbc:clickhouse://ip:port/tableName
    username: xxxx
    password: xxxx
    driver-class-name: ru.yandex.clickhouse.ClickHouseDriver
    initialSize: 10
    maxActive: 100
    minIdle: 10
    maxWait: 6000

mybatis:
  mapper-locations: classpath:mapping/*.xml
  type-aliases-package: com.xxxx.schedule.model
  configuration:
    log-impl: org.apache.ibatis.logging.stdout.StdOutImpl

四、Mapper.xml 及 mapper接口

<mapper namespace="com.xxxx.schedule.mapper.ext.UserBehaviorsUrlLogMapperExt">

  <sql id="Base_Column_List">
    (transfer_date, record_time, user, group, host_ip, dst_ip, serv, app, site, tm_type, net_action, url, dns, title, snapshot)
  sql>

  <insert id="insertUrlLog" parameterType="com.xxxx.schedule.model.UserBehaviorsUrlLog">
    INSERT INTO temp.xxzx_user_behaviors_url_log
    <include refid="Base_Column_List" />
    VALUES
    <foreach collection="records" item="item" index="index"
              separator=",">
      (#{item.transfer_date,jdbcType=DATE},#{item.record_time},#{item.user},#{item.group},#{item.host_ip},#{item.dst_ip},
      #{item.serv},#{item.app},#{item.site},#{item.tm_type},#{item.net_action},#{item.url},#{item.dns},#{item.title},#{item.snapshot})
    foreach>
  insert>
  
mapper>

@Mapper
public interface UserBehaviorsUrlLogMapperExt {

    /**
     * 读取行为日志信息，写入clickhouse
     * @param records
     * @return 落库条数
     */
   int insertUrlLog(@Param("records") List<UserBehaviorsUrlLog> records);

}

五、测试 TODO！

@Test
    void inserUrlLog(){
        List<UserBehaviorsUrlLog> list = new ArrayList<>();
        UserBehaviorsUrlLog urlLog = new UserBehaviorsUrlLog();
        Date date = null;
        try {
            date = new SimpleDateFormat("yyyy-MM-dd").parse("2021-05-23");
        } catch (ParseException e) {
            e.printStackTrace();
        }
        urlLog.setTransfer_date(date);

        urlLog.setRecord_time("23:04:08");
        urlLog.setUser("10.10.10.100");
        urlLog.setGroup("/");
        urlLog.setHost_ip("10.12.13.100");
        urlLog.setDst_ip("127.0.0.1");
        urlLog.setServ("游戏");
        urlLog.setApp("其他网页游戏[其他上传]");
        urlLog.setSite("未定义位置");
        urlLog.setTm_type("/PC/MAC PC");
        urlLog.setNet_action("拒绝");
        urlLog.setUrl("push.test.2345.cc/token");
        urlLog.setDns("push.test.2345.cc");
        urlLog.setTitle("测试");
        urlLog.setSnapshot("测试");

        list.add(urlLog);
        int i = urlLogMapperExt.insertUrlLog(list);
        System.out.println(i);
    }