Fastdfs-V5.11使用docker部署集群(X86)

一、 Fastdfs 介绍

FastDFS 是一款高性能的分布式文件系统，特别适合用于存储和管理大量的文件

二、部署 信息

• 使用 docker 部署 Fastdfs 集群

角色	服务器 IP	服务器系统	服务器配置	docker /docker-compose版本
storage	10.10.11.144	Centos 7.9	31G + 16C	19.03.8/1.29.2
storage	10.10.11.143	Centos 7.9	62G + 16C	19.03.8/1.29.2
storage	10.10.11.142	Centos 7.9	62G + 16C	19.03.15/v2.23.3
storage	10.10.11.141	Centos 7.9	31G + 16C	19.03.15/v2.23.3
tracker/storage	10.10.11.140	Centos 7.9	31G + 16C	19.03.15/v2.23.3
tracker/storage	10.10.11.139	Centos 7.9	31G + 16C	19.03.15/v2.23.3

• 默认 tracker 端口为 22122， storage 端口为 23000，文件访问端口为 8888

三、步骤

tracker/storage 机器的 compose 内容

version: '3.8'

networks:
  fastdfs:
    driver: bridge
    ipam:
      config:
        - subnet: 10.254.14.0/24
          gateway: 10.254.14.1
    name: fastdfs

x-logging: &default-logging
  driver: json-file
  options:
    max-size: "30m"
    max-file: "10"

services:
  tracker:
    image:  delron/fastdfs
    container_name: tracker
    hostname: tracker
    privileged: true
    restart: always
    network_mode: host
    volumes:
      - ./fastdfs/tracker:/var/fdfs 
      - /etc/localtime:/etc/localtime
    command: trcaker
    logging: *default-logging
  storage:
    image:  delron/fastdfs
    container_name: storage
    hostname: storage
    privileged: true
    restart: always
    network_mode: host
    volumes:
      - ./fastdfs/storage/data:/var/fdfs/data
      - ./fastdfs/storage/thumb:/var/fdfs/thumb
      - ./fastdfs/storage/logs:/var/fdfs/logs
      - ./fastdfs/storage/conf/storage.conf:/etc/fdfs/storage.conf
      - ./fastdfs/storage/conf/client.conf:/etc/fdfs/client.conf
      - /etc/localtime:/etc/localtime
    command: storage
    logging: *default-logging

storage 机器的 compose

version: '3.8'

networks:
  fastdfs:
    driver: bridge
    ipam:
      config:
        - subnet: 10.254.14.0/24
          gateway: 10.254.14.1
    name: fastdfs

x-logging: &default-logging
  driver: json-file
  options:
    max-size: "30m"
    max-file: "10"

services:
  storage:
    image:  delron/fastdfs
    container_name: storage
    hostname: storage
    privileged: true
    restart: always
    network_mode: host
    volumes:
      - ./fastdfs/storage/data:/var/fdfs/data
      - ./fastdfs/storage/thumb:/var/fdfs/thumb
      - ./fastdfs/storage/logs:/var/fdfs/logs
      - ./fastdfs/storage/conf/storage.conf:/etc/fdfs/storage.conf
      - ./fastdfs/storage/conf/client.conf:/etc/fdfs/client.conf
      - /etc/localtime:/etc/localtime
    command: storage
    logging: *default-logging

tracker 与 storage 启动

# 在142上查看 storage 日志，可以看到会输出其他节点连接成功的信息
docker  logs -f storage
storage  | [2024-12-10 17:01:35] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 10.10.11.140:22122, as a tracker client, my ip is 10.10.11.142
storage  | [2024-12-10 17:01:36] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 10.10.11.139:22122, as a tracker client, my ip is 10.10.11.142
storage  | [2024-12-10 17:01:38] INFO - file: tracker_client_thread.c, line: 1263, tracker server 10.10.11.139:22122, set tracker leader: 10.10.11.139:22122
storage  | [2024-12-10 17:01:38] INFO - file: storage_sync.c, line: 2732, successfully connect to storage server 10.10.11.143:23000
storage  | [2024-12-10 17:01:38] INFO - file: storage_sync.c, line: 2732, successfully connect to storage server 10.10.11.139:23000
storage  | [2024-12-10 17:01:38] INFO - file: storage_sync.c, line: 2732, successfully connect to storage server 10.10.11.140:23000
storage  | [2024-12-10 17:01:38] INFO - file: storage_sync.c, line: 2732, successfully connect to storage server 10.10.11.144:23000
storage  | [2024-12-10 17:01:38] INFO - file: storage_sync.c, line: 2732, successfully connect to storage server 10.10.11.141:23000

# 在139查看tracker日志
[2024-12-10 17:44:38] INFO - FastDFS v5.11, base_path=/var/fdfs, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=22122, bind_addr=, max_connections=256, accept_threads=1, work_threads=4, min_buff_size=8192, max_buff_size=131072, store_lookup=2, store_group=, store_server=0, store_path=0, reserved_storage_space=10.00%, download_server=0, allow_ip_count=-1, sync_log_buff_interval=10s, check_active_interval=120s, thread_stack_size=64 KB, storage_ip_changed_auto_adjust=1, storage_sync_file_max_delay=86400s, storage_sync_file_max_time=300s, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, use_storage_id=0, id_type_in_filename=ip, storage_id_count=0, rotate_error_log=0, error_log_rotate_time=00:00, rotate_error_log_size=0, log_file_keep_days=0, store_slave_file_use_link=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2024-12-10 17:44:39] INFO - file: tracker_relationship.c, line: 389, selecting leader...
[2024-12-10 17:44:40] INFO - file: tracker_relationship.c, line: 428, the tracker leader 10.10.11.140:22122

目录层级与配置文件

# 目录层级如下
fastdfs
    ├── storage
    │   ├── conf
    |   |   |—— client.conf
    |   |   |—— storage.conf
    │   ├── data
    │   ├── logs
    │   └── thumb
    └── tracker
        ├── data
        └── logs

client.conf

• 主要修改 tracker_server 的 ip 地址，如果有多个直接复制单行配置即可

connect_timeout=30
network_timeout=60
base_path=/var/fdfs
tracker_server=10.10.11.140:22122
tracker_server=10.10.11.139:22122
log_level=info
use_connection_pool = false
connection_pool_max_idle_time = 3600
load_fdfs_parameters_from_tracker=false
use_storage_id = false
storage_ids_filename = storage_ids.conf
http.tracker_server_port=80

storage.conf

• 主要修改 tracker_server 的 ip 地址，如果有多个直接复制单行配置即可
• 其中 22122 端口为 tracker 端口，23000 为 storage 端口，8888 为 storage 的 文件访问端口

disabled=false
bind_addr=
port=22122
connect_timeout=30
network_timeout=60
base_path=/var/fdfs
max_connections=256
accept_threads=1
work_threads=4
min_buff_size = 8KB
max_buff_size = 128KB
store_lookup=2
store_group=group2
store_server=0
store_path=0
download_server=0
reserved_storage_space = 10%
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
sync_log_buff_interval = 10
check_active_interval = 120
thread_stack_size = 64KB
storage_ip_changed_auto_adjust = true
storage_sync_file_max_delay = 86400
storage_sync_file_max_time = 300
use_trunk_file = false 
slot_min_size = 256
slot_max_size = 16MB
trunk_file_size = 64MB
trunk_create_file_advance = false
trunk_create_file_time_base = 02:00
trunk_create_file_interval = 86400
trunk_create_file_space_threshold = 20G
trunk_init_check_occupying = false
trunk_init_reload_from_binlog = false
trunk_compress_binlog_min_interval = 0
use_storage_id = false
storage_ids_filename = storage_ids.conf
id_type_in_filename = ip
store_slave_file_use_link = false
rotate_error_log = false
error_log_rotate_time=00:00
rotate_error_log_size = 0
log_file_keep_days = 0
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.server_port=8080
http.check_alive_interval=30
http.check_alive_type=tcp
http.check_alive_uri=/status.html
disabled=false
group_name=group1
bind_addr=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/var/fdfs
max_connections=1024
buff_size = 256KB
accept_threads=5
work_threads=25
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/var/fdfs
subdir_count_per_path=256
tracker_server=10.10.11.140:22122
tracker_server=10.10.11.139:22122
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
http.server_port=8888

查看集群信息

# 查看当前集群信息，任意节点的 storage 容器查看即可	
docker exec -it storage fdfs_monitor /etc/fdfs/client.conf  
# tracker_server_count = 2 表示 tracker 有两个
[2024-12-10 16:11:25] DEBUG - base_path=/var/fdfs, connect_timeout=30, network_timeout=60, tracker_server_count=2, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0
erver_count=2, server_index=0

tracker server is 10.10.11.139:22122  # 表示当前集群的Leader Tracker

group count: 1  # 表示共有一个 group

Group 1:  # 第一个 group 的信息
group name = group1  # group名称
disk total space = 289650 MB  # 磁盘总空间
disk free space = 172380 MB  # 磁盘空余空间
trunk free space = 0 MB 
storage server count = 2  # group内有两个 storage
active server count = 2  # 活动的storage有两个
storage server port = 23000  # storage的端口
storage HTTP port = 8888  # storage的文件访问端口
store path count = 1   # storage只挂载了一个目录 
subdir count per path = 256
current write server index = 0
current trunk file id = 0
。。。。。。
Storage 1:  # 表示第一个storage的信息
                id = 10.10.11.139
                ip_addr = 10.10.11.139 (master)  ACTIVE
。。。。。。
Storage 2:  # 第二个storage的信息
                id = 10.10.11.140
                ip_addr = 10.10.11.140 (slave1)  ACTIVE
。。。。。。
Storage 3:
                id = 10.10.11.141
                ip_addr = 10.10.11.141 (slave2)  ACTIVE
。。。。。。                
Storage 4:
                id = 10.10.11.142
                ip_addr = 10.10.11.142 (bogon)  ACTIVE
。。。。。。                
Storage 5:
                id = 10.10.11.143
                ip_addr = 10.10.11.143 (bogon)  ACTIVE
。。。。。。              
Storage 6:
                id = 10.10.11.144
                ip_addr = 10.10.11.144 (bogon)  ACTIVE

测试

# 通过访问tracker接口上传，但有可能会报错文件过大
curl -F "file=@./test.txt" http://10.10.11.140:22122/upload

# 通过容器内部命令上传
docker exec -it storage bash 
cd                      
echo 11111 > test.txt 
fdfs_test /etc/fdfs/client.conf upload test.txt

This is FastDFS client test program v5.11

Copyright (C) 2008, Happy Fish / YuQing

FastDFS may be copied only under the terms of the GNU General
Public License V3, which may be found in the FastDFS source kit.
Please visit the FastDFS Home Page http://www.csource.org/ 
for more detail.

[2024-12-10 16:48:07] DEBUG - base_path=/var/fdfs, connect_timeout=30, network_timeout=60, tracker_server_count=2, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0

tracker_query_storage_store_list_without_group: 
        server 1. group_name=, ip_addr=10.10.11.139, port=23000
        server 2. group_name=, ip_addr=10.10.11.140, port=23000

group_name=group1, ip_addr=10.10.11.139, port=23000
storage_upload_by_filename
group_name=group1, remote_filename=M00/00/00/CgoLi2dYAEeABV7XAAAABhc1CFI276.txt
source ip address: 10.10.11.139
file timestamp=2024-12-10 16:48:07
file size=6
file crc32=389351506
example file url: http://10.10.11.139/group1/M00/00/00/CgoLi2dYAEeABV7XAAAABhc1CFI276.txt
storage_upload_slave_by_filename
group_name=group1, remote_filename=M00/00/00/CgoLi2dYAEeABV7XAAAABhc1CFI276_big.txt
source ip address: 10.10.11.139
file timestamp=2024-12-10 16:48:07
file size=6
file crc32=389351506
example file url: http://10.10.11.139/group1/M00/00/00/CgoLi2dYAEeABV7XAAAABhc1CFI276_big.txt   # 加端口8888即可访问上传的文件

• 访问地址 http://10.10.11.139:8888/group1/M00/00/00/CgoLi2dYAEeABV7XAAAABhc1CFI276_big.txt

• fastdfs 集群直接的 storage 会自动同步数据，更换 ip 再次访问，也可以成功访问

测试集群扩容与缩减

● 扩容：目前有两种方法，扩容容量需要增加硬盘，挂载分区，扩容节点需要增加新的 group，修改新节点的配置后就可以加入集群，原有集群有数据后是无法直接增加节点的，除了上面增加 group 的方法就只能清除数据重新部署节点，所以推荐在部署集群之前最好先确认架构，防止后续因为架构调整带来的不必要的麻烦
● 缩减：由于集群会自动同步数据，要缩减直接直接关闭storage、修改nginx代理即可
PS：如果有其他方法也欢迎在评论区或私信反馈

nginx配置

upstream storage {
   server 10.10.11.139:23000;
   server 10.10.11.140:23000;
   server 10.10.11.141:23000;
   server 10.10.11.142:23000;
   server 10.10.11.143:23000;
   server 10.10.11.144:23000;
}

upstream storage_file {
   server 10.10.11.139:8888;
   server 10.10.11.140:8888;
   server 10.10.11.141:8888;
   server 10.10.11.142:8888;
   server 10.10.11.143:8888;
   server 10.10.11.144:8888;
}
upstream tracker {
   server 10.10.11.139:22122;
   server 10.10.11.140:22122;
   server 10.10.11.141:22122;
   server 10.10.11.142:22122;
   server 10.10.11.143:22122;
   server 10.10.11.144:22122;
}

server {
    listen 23001 ;
    server_name _;

    location / {
        proxy_set_header Host $http_host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header X-Forwarded-Host $http_host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_pass http://storage;
    }
}

server {
    listen 8889;
    server_name _;
    location / {
        proxy_set_header Host $http_host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header X-Forwarded-Host $http_host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_pass http://storage_file;
    }
}

server {
    listen 22123;
    server_name _;
    location / {
        proxy_set_header Host $http_host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header X-Forwarded-Host $http_host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_pass http://tracker;
    }
}