Doris镜像构建及容器编排

Doris镜像构建

以构建完成的 doris镜像 ponylee/doris，starrocks镜像 ponylee/starrocks。

entrypoint.sh 脚本文件

#!/bin/bash

set -e

initFe(){
    echo "start doris fe"
    [ -d "${FE_META}" ] || mkdir -p ${FE_META}
    sed -i "s|^[[:space:]]*\$LIMIT \$JAVA \$JAVA_OPTS org.apache.doris.PaloFe.*|    exec \$LIMIT \$JAVA \$JAVA_OPTS org.apache.doris.PaloFe \${HELPER} \"\$@\"|g" ${DORIS_DIR}/fe/bin/start_fe.sh
    echo "priority_networks = ${FE_NETWORK}" >> ${DORIS_DIR}/fe/conf/fe.conf
    exec bash ${DORIS_DIR}/fe/bin/start_fe.sh
}

initBe(){
    echo "start doris be"
    [ -d "${BE_DATA}" ] || mkdir -p ${BE_DATA}
    sed -i "s@^storage_root_path.*@storage_root_path = ${DORIS_DIR}/be/data@g" ${DORIS_DIR}/be/conf/be.conf
    echo "priority_networks = ${BE_NETWORK}" >> ${DORIS_DIR}/be/conf/be.conf
    sed -i "s|^[[:space:]]*\$LIMIT \${DORIS_HOME}/lib/palo_be.*|    exec \$LIMIT \${DORIS_HOME}/lib/palo_be \"\$@\"|g" ${DORIS_DIR}/be/bin/start_be.sh

    #ENV FE_HOST, add be to fe
    #ENV BE_HOST
    COUNT=$(mysql -h${FE_HOST} -P9030 -uroot -e"SHOW PROC '/backends'\G;" | grep ${BE_HOST} | wc -l)
    if [ $COUNT -lt 1  ];
    then
        mysql -h${FE_HOST} -P9030 -uroot -e"ALTER SYSTEM ADD BACKEND \"${BE_HOST}:9050\";"
    fi
    exec bash ${DORIS_DIR}/be/bin/start_be.sh
}

printUsage() {
    echo -e "Usage: [ fe | be ]\n"
    printf "%-13s:  %s\n" "fe" "Doris master."
    printf "%-13s:  %s\n" "be" "Doris worker."
}

case "$1" in
    (fe)
        initFe
    ;;
    (be)
        initBe
    ;;
    (help)
        printUsage
        exit 1
    ;;
    (*)
        printUsage
        exit 1
    ;;
esac

Dockerfile

FROM centos:7
MAINTAINER pony

WORKDIR /opt

ADD ./DORIS-0.11.33.1-release.tar.gz ./jdk1.8.tar.gz /opt/

ENV DORIS_DIR /opt/output
ENV FE_LOG ${DORIS_DIR}/fe/log
ENV FE_META ${DORIS_DIR}/fe/palo-meta
ENV FE_TMP ${DORIS_DIR}/fe/temp_dir

ENV BE_LOG ${DORIS_DIR}/be/log
ENV BE_DATA ${DORIS_DIR}/be/data

ENV JAVA_HOME /opt/jdk1.8.0_271
ENV PATH $PATH:$JAVA_HOME/bin

# set the time zone
RUN rm -f /etc/localtime; ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

# install mysql client
RUN yum install -y mariadb.x86_64 mariadb-libs.x86_64 && yum clean all

COPY ./startup.sh /opt/startup.sh

VOLUME [ "${FE_LOG}","${FE_META}","${FE_TMP}","${BE_LOG}","${BE_DATA}" ]
EXPOSE 8030 8040

ENTRYPOINT ["/bin/bash","-x","entrypoint.sh"]

#CMD ["be/fe"]

其中DORIS-0.11.33.1-release.tar.gz、jdk1.8.tar.gz 根据实际需要去官网下载相应的版本

构建镜像

docker build -t doris:v1 ./

编排文件

doris-docker-compose.yml

version: '3'
services:

  fe:
    image: pony/doris:v4
    container_name: fe
    hostname: fe
    restart: always
    command: ["fe"]
    environment:
      FE_NETWORK: 192.17.1.30/24
    deploy:
      resources:
        limits:
          cpus: '2.00'
          memory: 5G
        reservations:
          cpus: '0.25'
          memory: 100M
    volumes:
      - /root/doris/fe/log:/opt/output/fe/log:rw
      - /root/doris/fe/palo-meta:/opt/output/fe/palo-meta:rw
      - /root/doris/fe/temp_dir:/opt/output/fe/temp_dir:rw
    ports:
      - "8030:8030"
    networks:
      bdpnet:
        ipv4_address: 192.17.1.30

  be01:
    image: pony/doris:v4
    container_name: be01
    hostname: be01
    restart: always
    command: [ "be" ]
    depends_on:
      - fe
    environment:
      FE_HOST: fe
      BE_HOST: be01
      BE_NETWORK: 192.17.1.31/24
    deploy:
      resources:
        limits:
          cpus: '2.00'
          memory: 5G
        reservations:
          cpus: '0.25'
          memory: 100M
    volumes:
      - /root/doris/be01/log:/opt/output/be/log:rw
      - /root/doris/be01/data:/opt/output/be/data:rw
    ports:
      - "8041:8040"
    networks:
      bdpnet:
        ipv4_address: 192.17.1.31

  be02:
    image: pony/doris:v4
    container_name: be02
    hostname: be02
    restart: always
    command: [ "be" ]
    depends_on:
      - fe
    environment:
      FE_HOST: fe
      BE_HOST: be02
      BE_NETWORK: 192.17.1.32/24
    deploy:
      resources:
        limits:
          cpus: '2.00'
          memory: 5G
        reservations:
          cpus: '0.25'
          memory: 100M
    volumes:
      - /root/doris/be02/log:/opt/output/be/log:rw
      - /root/doris/be02/data:/opt/output/be/data:rw
    ports:
      - "8042:8040"
    networks:
      bdpnet:
        ipv4_address: 192.17.1.32

#docker network create -d bridge --subnet 192.17.1.0/24 --gateway 192.17.1.1 bdpnet
networks:
  bdpnet:
    external: true

#volumes:
#  fe:
#  be01:
#  be02:
  

启动doris集群

docker-compose -f doris-docker-compose.yml up -d

k8s编排

fe/master编排

apiVersion: v1
kind: Service
metadata:
  name: bdp-doris-fe
  namespace: bdp
  labels:
    k8s-app: bdp-doris-fe
spec:
  ports:
    - name: tcp-8030-8030
      protocol: TCP
      port: 8030
      targetPort: 8030
    - name: tcp-9030-9030
      protocol: TCP
      port: 9030
      targetPort: 9030
    - name: tcp-9020-9020
      protocol: TCP
      port: 9020
      targetPort: 9020
  clusterIP: 11.254.120.1
  selector:
    k8s-app: bdp-doris-fe
---
# 创建Headless Service
apiVersion: v1
kind: Service
metadata:
  name: bdp-doris-fe-hs
  namespace: bdp
  labels:
    k8s-app: bdp-doris-fe
spec:
  ports:
    - name: tcp-8030-8030-fe-hs
      protocol: TCP
      port: 8030
      targetPort: 8030
    - name: tcp-9030-9030-fe-hs
      protocol: TCP
      port: 9030
      targetPort: 9030
    - name: tcp-9020-9020-fe-hs
      protocol: TCP
      port: 9020
      targetPort: 9020
  clusterIP: None
  selector:
    k8s-app: bdp-doris-fe
---
# 创建pv，使用CBS存储
apiVersion: v1
kind: PersistentVolume
metadata:
  name: bdp-doris-fe-pv  ##pv名称
spec:
  capacity:
    storage: 500Mi ## pv大小
  accessModes:
    - ReadWriteOnce ## pv读写类型，目前支持ReadWriteOnce和ReadOnlyMany
  mountOptions:
    - rw ##挂载类型有只读（ro）,读写{rw}，挂载类型和accessModes要对应起来
    - discard
  persistentVolumeReclaimPolicy: Retain
  storageClassName: mysql-local-storage ## 使用和mysql相同的sc
  local:
    path: /mnt/fe
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - fe
---
#pvc
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: bdp-doris-fe-pvc ##pvc名称
  namespace: bdp
spec:                ##pv的筛选条件
  accessModes:
     - ReadWriteOnce ## pv读写模式
  volumeMode: Filesystem
  volumeName: bdp-doris-fe-pv  ##指定pv名称，此为可选字段，当指定特定的pv名称时，只有对应的名称的pv能被bound
  resources:
     requests:
       storage: 150Gi  ##pv大小
---
# 创建StatefulSets
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: bdp-doris-fe
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: bdp-doris-fe
  serviceName: bdp-doris-fe-hs
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: OrderedReady
  template:
    metadata:
      labels:
        app: dbp-doris
        k8s-app: bdp-doris-fe
    spec:
      nodeSelector:
        kubernetes.io/hostname: 192.168.0.15
      restartPolicy: Always
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: bdp-doris-fe
        image: harbor.dcos.xixian.unicom.local/bdp/doris:v4
        imagePullPolicy: Always
        args:
        - fe
        env:
        - name: FE_NETWORK
          value: 192.168.0.15/24  # fe 的 hostname，需要根据实际情况修改
        resources:
          limits:
            cpu: "4"
            memory: "8Gi"
          requests:
            cpu: 250m
            memory: "200Mi"
        ports:
        - containerPort: 8030
          name: fe-http-port
        - containerPort: 9030
          name: fe-query-port
        - containerPort: 9020
          name: fe-rpc-port
        volumeMounts:
        - mountPath: /opt/output/fe/palo-meta
          name: bdp-doris-fe-pvc
          subPath: palo-meta
        - mountPath: /opt/output/fe/temp_dir
          name: bdp-doris-fe-pvc
          subPath: temp_dir
        - mountPath: /opt/output/fe/log
          name: bdp-doris-fe-pvc
          subPath: log
      volumes:
      - name: bdp-doris-fe-pvc
        persistentVolumeClaim:
          claimName: bdp-doris-fe-pvc

be/work编排

apiVersion: v1
kind: Service
metadata:
  name: bdp-doris-be01-hs
  namespace: bdp
  labels:
    k8s-app: bdp-doris-be01
spec:
  ports:
    - name: tcp-8040-8040-be01-hs
      protocol: TCP
      port: 8040
      targetPort: 8040
    - name: tcp-9050-9050-be01-hs
      protocol: TCP
      port: 9050
      targetPort: 9050
    - name: tcp-9060-9060-be01-hs
      protocol: TCP
      port: 9060
      targetPort: 9060
    - name: tcp-9070-9070-be01-hs
      protocol: TCP
      port: 9070
      targetPort: 9070
  clusterIP: None
  selector:
    k8s-app: bdp-doris-be01
---
apiVersion: v1
kind: Service
metadata:
  name: bdp-doris-be02-hs
  namespace: bdp
  labels:
    k8s-app: bdp-doris-be02
spec:
  ports:
    - name: tcp-8040-8040-be02-hs
      protocol: TCP
      port: 8040
      targetPort: 8040
    - name: tcp-9050-9050-be02-hs
      protocol: TCP
      port: 9050
      targetPort: 9050
    - name: tcp-9060-9060-be02-hs
      protocol: TCP
      port: 9060
      targetPort: 9060
    - name: tcp-9070-9070-be02-hs
      protocol: TCP
      port: 9070
      targetPort: 9070
  clusterIP: None
  selector:
    k8s-app: bdp-doris-be02
---
apiVersion: v1
kind: Service
metadata:
  name: bdp-doris-be03-hs
  namespace: bdp
  labels:
    k8s-app: bdp-doris-be03
spec:
  ports:
    - name: tcp-8040-8040-be03-hs
      protocol: TCP
      port: 8040
      targetPort: 8040
    - name: tcp-9050-9050-be03-hs
      protocol: TCP
      port: 9050
      targetPort: 9050
    - name: tcp-9060-9060-be03-hs
      protocol: TCP
      port: 9060
      targetPort: 9060
    - name: tcp-9070-9070-be03-hs
      protocol: TCP
      port: 9070
      targetPort: 9070
  clusterIP: None
  selector:
    k8s-app: bdp-doris-be03
---
# be01 pv
apiVersion: v1
kind: PersistentVolume
metadata:
  name: bdp-doris-be01-pv  ##pv名称
spec:
  capacity:
    storage: 500Mi ## pv大小
  accessModes:
    - ReadWriteOnce ## pv读写类型，目前支持ReadWriteOnce和ReadOnlyMany
  mountOptions:
    - rw ##挂载类型有只读（ro）,读写{rw}，挂载类型和accessModes要对应起来
    - discard
  persistentVolumeReclaimPolicy: Retain
  storageClassName: mysql-local-storage ## 使用和mysql相同的sc
  local:
    path: /mnt/be01
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - be01
---
# be02 pv
apiVersion: v1
kind: PersistentVolume
metadata:
  name: bdp-doris-be02-pv  ##pv名称
spec:
  capacity:
    storage: 500Mi ## pv大小
  accessModes:
    - ReadWriteOnce ## pv读写类型，目前支持ReadWriteOnce和ReadOnlyMany
  mountOptions:
    - rw ##挂载类型有只读（ro）,读写{rw}，挂载类型和accessModes要对应起来
    - discard
  persistentVolumeReclaimPolicy: Retain
  storageClassName: mysql-local-storage ## 使用和mysql相同的sc
  local:
    path: /mnt/be02
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - be02
---
# be03 pv
apiVersion: v1
kind: PersistentVolume
metadata:
  name: bdp-doris-be03-pv  ##pv名称
spec:
  capacity:
    storage: 500Mi ## pv大小
  accessModes:
    - ReadWriteOnce ## pv读写类型，目前支持ReadWriteOnce和ReadOnlyMany
  mountOptions:
    - rw ##挂载类型有只读（ro）,读写{rw}，挂载类型和accessModes要对应起来
    - discard
  persistentVolumeReclaimPolicy: Retain
  storageClassName: mysql-local-storage ## 使用和mysql相同的sc
  local:
    path: /mnt/be03
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - be03
---
# be01 pvc
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: bdp-doris-be01-pvc ##pvc名称
  namespace: bdp
spec:                ##pv的筛选条件
  accessModes:
     - ReadWriteOnce ## pv读写模式
  volumeMode: Filesystem
  volumeName: bdp-doris-be01-pv  ##指定pv名称，此为可选字段，当指定特定的pv名称时，只有对应的名称的pv能被bound
  resources:
     requests:
       storage: 500Gi  ##pv大小
---
# be02 pvc
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: bdp-doris-be02-pvc ##pvc名称
  namespace: bdp
spec:                ##pv的筛选条件
  accessModes:
     - ReadWriteOnce ## pv读写模式
  volumeMode: Filesystem
  volumeName: bdp-doris-be02-pv  ##指定pv名称，此为可选字段，当指定特定的pv名称时，只有对应的名称的pv能被bound
  resources:
     requests:
       storage: 500Gi  ##pv大小
---
# be03 pvc
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: bdp-doris-be03-pvc ##pvc名称
  namespace: bdp
spec:                ##pv的筛选条件
  accessModes:
     - ReadWriteOnce ## pv读写模式
  volumeMode: Filesystem
  volumeName: bdp-doris-be03-pv  ##指定pv名称，此为可选字段，当指定特定的pv名称时，只有对应的名称的pv能被bound
  resources:
     requests:
       storage: 500Gi  ##pv大小
---
# be01
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: bdp-doris-be01
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: bdp-doris-be01
  serviceName: bdp-doris-be01-hs
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: OrderedReady
  template:
    metadata:
      labels:
        app: dbp-doris
        k8s-app: bdp-doris-be01
    spec:
      nodeSelector:
        kubernetes.io/hostname: 192.168.0.4
      restartPolicy: Always
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: bdp-doris-be01
        image:  harbor.dcos.xixian.unicom.local/bdp/doris:v4
        imagePullPolicy: Always
        args:
        - be
        env:
        - name: FE_HOST
          value: 192.168.0.15  # fe 的 hostname，需要根据实际情况修改
        - name: BE_HOST
          value: 192.168.0.4 # be 的 hostname，需要根据实际情况修改
        - name: BE_NETWORK
          value: 192.168.0.4/24  # 指定 fe 通信的网卡
        resources:
          limits:
            cpu: "8"
            memory: "16Gi"
          requests:
            cpu: 250m
            memory: "100Mi"
        ports:
        - containerPort: 8040
          name: be-ws-port
        - containerPort: 9050
          name: be-hs-port
        - containerPort: 9060
          name: be-port
        - containerPort: 9070
          name: be-rpc-port
        volumeMounts:
        - mountPath: /opt/output/be/data
          name: bdp-doris-be01-pvc
          subPath: data
        - mountPath: /opt/output/be/log
          name: bdp-doris-be01-pvc
          subPath: log
      volumes:
      - name: bdp-doris-be01-pvc
        persistentVolumeClaim:
          claimName: bdp-doris-be01-pvc
---
# be02
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: bdp-doris-be02
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: bdp-doris-be02
  serviceName: bdp-doris-be02-hs
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: OrderedReady
  template:
    metadata:
      labels:
        app: dbp-doris
        k8s-app: bdp-doris-be02
    spec:
      nodeSelector:
        kubernetes.io/hostname: 192.168.0.12
      restartPolicy: Always
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: bdp-doris-be02
        image: harbor.dcos.xixian.unicom.local/bdp/doris:v4
        imagePullPolicy: Always
        args:
        - be
        env:
        - name: FE_HOST
          value: 192.168.0.15  # fe 的 hostname，需要根据实际情况修改
        - name: BE_HOST
          value: 192.168.0.12 # be 的 hostname，需要根据实际情况修改
        - name: BE_NETWORK
          value: 192.168.0.12/24  # 指定 fe 通信的网卡
        resources:
          limits:
            cpu: "8"
            memory: "16Gi"
          requests:
            cpu: 250m
            memory: "100Mi"
        ports:
        - containerPort: 8040
          name: be-ws-port
        - containerPort: 9050
          name: be-hs-port
        - containerPort: 9060
          name: be-port
        - containerPort: 9070
          name: be-rpc-port
        volumeMounts:
        - mountPath: /opt/output/be/data
          name: bdp-doris-be02-pvc
          subPath: data
        - mountPath: /opt/output/be/log
          name: bdp-doris-be02-pvc
          subPath: log
      volumes:
      - name: bdp-doris-be02-pvc
        persistentVolumeClaim:
          claimName: bdp-doris-be02-pvc
---
# be03
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: bdp-doris-be03
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: bdp-doris-be03
  serviceName: bdp-doris-be03-hs
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: OrderedReady
  template:
    metadata:
      labels:
        app: dbp-doris
        k8s-app: bdp-doris-be03
    spec:
      nodeSelector:
        kubernetes.io/hostname: 192.168.0.13
      restartPolicy: Always
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: bdp-doris-be03
        image: harbor.dcos.xixian.unicom.local/bdp/doris:v4
        imagePullPolicy: Always
        args:
        - be
        env:
        - name: FE_HOST
          value: 192.168.0.15  # fe 的 hostname，需要根据实际情况修改
        - name: BE_HOST
          value: 192.168.0.13 # be 的 hostname，需要根据实际情况修改
        - name: BE_NETWORK
          value: 192.168.0.13/24  # 指定 fe 通信的网卡
        resources:
          limits:
            cpu: "8"
            memory: "16Gi"
          requests:
            cpu: 250m
            memory: "100Mi"
        ports:
        - containerPort: 8040
          name: be-ws-port
        - containerPort: 9050
          name: be-hs-port
        - containerPort: 9060
          name: be-port
        - containerPort: 9070
          name: be-rpc-port
        volumeMounts:
        - mountPath: /opt/output/be/data
          name: bdp-doris-be03-pvc
          subPath: data
        - mountPath: /opt/output/be/log
          name: bdp-doris-be03-pvc
          subPath: log
      volumes:
      - name: bdp-doris-be03-pvc
        persistentVolumeClaim:
          claimName: bdp-doris-be03-pvc

关键技术点

优先网络

在Kubernetes多网卡等容器化编排环境中因为有多网卡的存在，通过固定IP的形式部署Doris集群，同一个主机可能存在多个不同的 ip。当前Doris实例并不能自动识别可用 IP。所以当遇到部署主机上有多个IP时，必须通过优先网络（priority_networks）来强制指定正确的 IP（即容器编排是传入的IP地址）。否则doris实例会随机去绑定一个ip，这样以来在当前Doris实例故障恢复时就不能再与当前Doris集群建立连接。

集群实例宕机恢复机制

通过强制指定优先网络和磁盘挂载来实现Doris集群中的实例宕机故障恢复

注意

第一次启动fe一定要用–helper启动，否则启动不成功，无法加入集群，必须借助已经启动的FE节点（master或者follower）来扩容新的节点,helper 不能指向 新FE节点 自身
当新的fe第一次启动失败，或无法加入集群时。修改配置后，一定要先清空fe的meta路径，然后再重启fe，參考。