pgspider clickhouse fdw v2 版本

此版本是基于clickhouse 的http 以及binary 协议更灵活高效

环境准备

• dockefile

FROM dalongrong/pgspider:base as build

WORKDIR /app

RUN apt-get update && apt-get install -y libcurl4-openssl-dev automake autoconf libtool pkg-config  uuid-dev  cmake git build-essential gcc openssl

RUN git clone https://github.com/adjust/clickhouse_fdw.git /app/postgresql-11.6/contrib/clickhousedb_fdw

RUN cd /app/postgresql-11.6/contrib/clickhousedb_fdw && mkdir build && cd build \ 

    && cmake .. && make && make install

​

FROM debian:stretch-slim

ENV GOSU_VERSION 1.11

RUN apt-get update && apt-get install -y  uuid-dev wget openssl libreadline-dev libcurl4-openssl-dev

# explicitly set user/group IDs

RUN set -eux; \

    groupadd -r postgres --gid=999; \

# https://salsa.debian.org/postgresql/postgresql-common/blob/997d842ee744687d99a2b2d95c1083a2615c79e8/debian/postgresql-common.postinst#L32-35

    useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \

# also create the postgres user's home directory with appropriate permissions

# see https://github.com/docker-library/postgres/issues/274

    mkdir -p /var/lib/postgresql; \

    chown -R postgres:postgres /var/lib/postgresql

​

RUN wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \

   && chmod +x /usr/local/bin/gosu \

   && gosu nobody true

# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default

RUN set -eux; \

   if [ -f /etc/dpkg/dpkg.cfg.d/docker ]; then \

   # if this file exists, we're likely in "debian:xxx-slim", and locales are thus being excluded so we need to remove that exclusion (since we need locales)

   grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; \

   sed -ri '/\/usr\/share\/locale/d' /etc/dpkg/dpkg.cfg.d/docker; \

   ! grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; \

   fi; \

   apt-get update; apt-get install -y locales; rm -rf /var/lib/apt/lists/*; \

   localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8

ENV LANG en_US.utf8

​

# install "nss_wrapper" in case we need to fake "/etc/passwd" and "/etc/group" (especially for OpenShift)

# https://github.com/docker-library/postgres/issues/359

# https://cwrap.org/nss_wrapper.html

RUN set -eux; \

   apt-get update; \

   apt-get install -y --no-install-recommends libnss-wrapper; \

   rm -rf /var/lib/apt/lists/*

​

RUN mkdir /docker-entrypoint-initdb.d && mkdir -p /opt/clickhousedb

COPY  --from=build  /usr/local/pgspider /usr/local/pgspider

​

RUN sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/pgspider/share/postgresql/postgresql.conf.sample; \

   grep -F "listen_addresses = '*'" /usr/local/pgspider/share/postgresql/postgresql.conf.sample

RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql  && chmod 2777 /var/run/postgresql

ENV PATH $PATH:/usr/local/pgspider/bin

ENV PGDATA /var/lib/postgresql/data

RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"

VOLUME /var/lib/postgresql/data

COPY docker-entrypoint.sh /usr/local/bin/

RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat

ENTRYPOINT ["docker-entrypoint.sh"]

​

EXPOSE 5432

CMD ["postgres"]

• docker-compose 文件

version: "3"

services:

    server:

     image: yandex/clickhouse-server

     ports:

     - "8123:8123"

     - "9000:9000"

     - "9009:9009"

     ulimits:

      nproc: 65535

      nofile:

       soft: 262144

       hard: 262144

    client:

      image: yandex/clickhouse-client

      command: ['--host', 'server']

    pgspider:

      image: dalongrong/pgspider:clickhourse2

      ports: 

      - "5432:5432"

      environment: 

      - "POSTGRES_PASSWORD=dalong"

• 启动&&数据初始化

docker-compose up -d

CREATE TABLE wikistat

(

    project String,

    subproject String,

    hits UInt64,

    size UInt64

) ENGINE = Log;

​

导入数据：

serverhost=192.168.0.103 // replace with your hostid

docker run -i yandex/clickhouse-client  --format_csv_delimiter="|" --host ${serverhost} --query="INSERT INTO default.wikistat FORMAT CSV" < ./data/info.csv

使用

• 创建扩展以及用户映射

CREATE EXTENSION clickhouse_fdw;

CREATE SERVER clickhouse_svr FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'default', host 'server');

CREATE USER MAPPING FOR postgres SERVER clickhouse_svr

       OPTIONS (user 'default', password '');

• 导入schema

IMPORT FOREIGN SCHEMA "default" FROM SERVER clickhouse_svr INTO public;

• 查询

select * from wikistat;

• 效果

 

 

说明

https://github.com/adjust/clickhouse_fdw 的fdw 相比Percona有很多提升，是一个不错的选择

参考资料

https://github.com/Percona-Lab/clickhousedb_fdw
https://github.com/adjust/clickhouse_fdw
https://github.com/rongfengliang/pgspider-docker