工作随笔 - docker-compose搭建多主机分布式minio

根据官网介绍http://docs.minio.org.cn/docs/master/deploy-minio-on-docker-swarm，多主机minio只有介绍swarm的模式，但是单独为minio搭建swarm有点麻烦，所以改造了官网docker-compose模式http://docs.minio.org.cn/docs/master/deploy-minio-on-docker-compose单机模式为多机模式

服务器信息如下：

服务器	IP	安装组件
node1	192.168.1.1	minio1、nginx
node2	192.168.1.2	minio2、nginx
node3	192.168.1.3	minio3、nginx

安装配置步骤如下（假设已经安装docker，本处使用docker-ce 19.03.12）：

1. docker下载镜像（如果服务器能够访问公网，则忽略此步骤）

docker pull quay.io/minio/minio:RELEASE.2021-10-23T03-28-24Z
docker pull nginx:1.19.2-alpine

2. 下载docker-compose (本处使用docker-compose version 1.29.2，docker-compose需要支持语法3.7)

# 增加执行权限
chmod +x docker-compose

3. 创建docker-compose.yaml(按需修改IP)

version: '3.7'

# Settings and configurations that are common for all containers
# minio节点之间默认使用9000来连通，所以容器把9000暴露出来，9001是console端口，每个节点设置两块磁盘
x-minio-common: &minio-common
  image: quay.io/minio/minio:RELEASE.2021-10-23T03-28-24Z
  command: server --console-address ":9001" http://minio{1...3}/data{1...2}
  expose:
    - "9000"
    - "9001"
# 增加host映射，以便三个节点之间通过域名连通
  extra_hosts:
    minio1: 192.168.1.1
    minio2: 192.168.1.2
    minio3: 192.168.1.3
  depends_on:
    - nginx
  environment:
    MINIO_ROOT_USER: minio
    MINIO_ROOT_PASSWORD: minio123
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
    interval: 30s
    timeout: 20s
    retries: 5

# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
# 数据盘挂载目录，按需修改
services:
  minio1:
    <<: *minio-common
    container_name: minio1
    hostname: minio1
    volumes:
      - ./minio_data/data1-1:/data1
      - ./minio_data/data1-2:/data2
    ports:
      - "9000:9000"
      - "9001:9001"

  minio2:
    <<: *minio-common
    container_name: minio2
    hostname: minio2
    volumes:
      - ./minio_data/data2-1:/data1
      - ./minio_data/data2-2:/data2
    ports:
      - "9000:9000"
      - "9001:9001"

  minio3:
    <<: *minio-common
    container_name: minio3
    hostname: minio3
    volumes:
      - ./minio_data/data3-1:/data1
      - ./minio_data/data3-2:/data2
    ports:
      - "9000:9000"
      - "9001:9001"
# 三个节点都安装nginx，并且负载到三个节点，nginx暴露端口按需修改19000/19001
  nginx:
    image: nginx:1.19.2-alpine
    container_name: nginx-minio
    hostname: nginx
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "19000:9000"
      - "19001:9001"
    extra_hosts:
      minio1: 192.168.1.1
      minio2: 192.168.1.2
      minio3: 192.168.1.3

4. 创建nginx配置如下：

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;

    # include /etc/nginx/conf.d/*.conf;

    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
    }

    upstream console {
        ip_hash;
        server minio1:9001;
        server minio2:9001;
        server minio3:9001;
    }

    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://minio;
        }
    }

    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;

            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;

            proxy_connect_timeout 300;

            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";

            chunked_transfer_encoding off;

            proxy_pass http://console;
        }
    }
}

5. 三个节点分别启动

# 拉包
./docker-compose pull
# 后台启动，三台对应启动各自节点minio1/minio2/minio3 (切勿单台启动多个)
./docker-compose up -d minio1
./docker-compose up -d minio2
./docker-compose up -d minio3

6. 验证，登陆 http://192.168.1.1:19001/，账号密码 minio/minio123