分布式文件系统FastDFS详解

个人博客请访问 http://www.x0100.top  

01

序

  FastDFS是一个开源的轻量级分布式文件系统，它对文件进行管理，功能包括：文件存储、文件同步、文件访问（文件上传、文件下载）等，解决了大容量存储和负载均衡的问题。特别适合以文件为载体的在线服务，如相册网站、视频网站等等。
 FastDFS为互联网量身定制，充分考虑了冗余备份、负载均衡、线性扩容等机制，并注重高可用、高性能等指标，使用FastDFS很容易搭建一套高性能的文件服务器集群提供文件上传、下载等服务。

 

02

FastDFS介绍

  是一款开源的轻量级分布式文件系统，纯C实现，支持linux、freeBSD等unix系统，这种分布式存储更适合4kb--500MB之间大小的文件存储。

三种角色：

1）客户端

2）tracker追踪服务器

3）storage存储服务器

客户端请求——追踪服务器——给客户端分配一个组——最终客户端将数据写入存储组中

追踪服务器：主要负责调度工作，在访问时起到负载均衡的作用，在内存中记录集群中group和storage服务器的状态信息，是连接客户端与group和storage枢纽

Storage server：存储服务器，文件和文件属性都保存在存储服务器上

FastDFS上传机制：

 

FastDFS下载机制：

 

03

FastDFS安装准备

 

利用两台虚拟机进行模拟环境

fastdfs-1-1  192.168.1.128

fastdfs-1-1  192.168.1.129

yum install git -y

下载官网：https://github.com/happyfish100

 

下载安装依赖包libfastcommon.git

[root@fastdfs-1-1 src]# git clone https://github.com/happyfish100/libfastcommon.git

[root@fastdfs-1-1 src]# cd libfastcommon/

[root@fastdfs-1-1libfastcommon]# ll

total 36

drwxr-xr-x 2 root root 4096 Mar  6 07:13 doc

-rw-r--r-- 1 root root 7939 Mar  6 07:13 HISTORY

-rw-r--r-- 1 root root  566 Mar  6 07:13 INSTALL

-rw-r--r-- 1 root root 1494 Mar  6 07:13 libfastcommon.spec

-rwxr-xr-x 1 root root 3099 Mar  6 07:13 make.sh

drwxr-xr-x 2 root root 4096 Mar  6 07:13 php-fastcommon

-rw-r--r-- 1 root root 2763 Mar  6 07:13 README

drwxr-xr-x 3 root root 4096 Mar  6 07:13 src

[root@fastdfs-1-1 libfastcommon]# ./make.sh

[root@fastdfs-1-1 libfastcommon]# ./make.sh install

 

04

FastDFS安装布署

下载安装fastdfs服务

[root@fastdfs-1-1 src]# wget https://github.com/happyfish100/fastdfs/archive/V5.09.tar.gz

[root@fastdfs-1-1 src]# ll

total 336

-rw-r--r-- 1 root root 335132 Mar  6 07:30 fastdfs-5.09.tar.gz

drwxr-xr-x 6 root root   4096 Mar  6 07:23 libfastcommon

[root@fastdfs-1-1 src]# tar zxf fastdfs-5.09.tar.gz

[root@fastdfs-1-1 src]# cd fastdfs-5.09

[root@fastdfs-1-1 fastdfs-5.09]# ./make.sh

[root@fastdfs-1-1 fastdfs-5.09]# ./make.sh install

[root@fastdfs-1-1 src]# ll /etc/fdfs/

total 24

-rw-r--r-- 1 root root 1461 Mar  6 07:32 client.conf.sample

-rw-r--r-- 1 root root 7927 Mar  6 07:32 storage.conf.sample

-rw-r--r-- 1 root root  105 Mar  6 07:32 storage_ids.conf.sample

-rw-r--r-- 1 root root 7318 Mar  6 07:32 tracker.conf.sample

[root@fastdfs-1-1 src]# ll /usr/bin/fdfs_

fdfs_appender_test  fdfs_download_file fdfs_test1

fdfs_appender_test1  fdfs_file_info  fdfs_trackerd

fdfs_append_file  fdfs_monitor fdfs_upload_appender

fdfs_crc32  fdfs_storaged   fdfs_upload_file

fdfs_delete_file  fdfs_test           

修改默认路径

[root@fastdfs-1-1 src]# sed -i 's#/usr/local/bin#/usr/bin#g' /etc/init.d/fdfs_storaged

[root@fastdfs-1-1 src]# sed -i 's#/usr/local/bin#/usr/bin#g' /etc/init.d/fdfs_trackerd

创建数据目录

[root@fastdfs-1-1 fdfs]# mkdir /fdfs/{tracker,storage} -p

[root@fastdfs-1-1 fdfs]# mkdir /fdfs/storage/base -p

[root@fastdfs-1-1 fdfs]# mkdir /fdfs/storage/store -p

[root@fastdfs-1-1 fdfs]# ll /fdfs/

total 8

drwxr-xr-x 2 root root 4096 Mar  6 07:42 storage

drwxr-xr-x 2 root root 4096 Mar  6 07:42 tracker

修改配置文件

[root@fastdfs-1-1 fdfs]# egrep -v "^#|^$" tracker.conf

disabled=false

bind_addr=  #绑定的IP

port=22122  #端口

connect_timeout=30

network_timeout=60

base_path=/fdfs/tracker #存储路径

max_connections=256

accept_threads=1

work_threads=4

min_buff_size = 8KB

max_buff_size = 128KB

store_lookup=2   #存储的模式

store_group=group2 # which group to upload file

store_server=0   # which storage server to upload file

store_path=0    

download_server=0  # which storage server to download file

reserved_storage_space = 10%  

# a group <= reserved_storage_space,保留的空间大小

log_level=info

run_by_group=

run_by_user=

allow_hosts=*

sync_log_buff_interval = 10

check_active_interval = 120

thread_stack_size = 64KB

storage_ip_changed_auto_adjust = true

storage_sync_file_max_delay = 86400

storage_sync_file_max_time = 300

use_trunk_file = false

slot_min_size = 256

slot_max_size = 16MB

trunk_file_size = 64MB

trunk_create_file_advance = false

trunk_create_file_time_base = 02:00

trunk_create_file_interval = 86400

trunk_create_file_space_threshold = 20G

trunk_init_check_occupying = false

trunk_init_reload_from_binlog = false

trunk_compress_binlog_min_interval = 0

use_storage_id = false

storage_ids_filename = storage_ids.conf

id_type_in_filename = ip

store_slave_file_use_link = false

rotate_error_log = false

error_log_rotate_time=00:00

rotate_error_log_size = 0

log_file_keep_days = 0

use_connection_pool = false

connection_pool_max_idle_time = 3600

http.server_port=8080

http.check_alive_interval=30

http.check_alive_type=tcp

http.check_alive_uri=/status.html

[root@fastdfs-1-2 fdfs]# /etc/init.d/fdfs_trackerd start

Starting FastDFS tracker server:

[root@fastdfs-1-2 fdfs]# lsof -i :22122

COMMAND  PID USER   FD   TYPE DEVICE SIZE/OFF NODE NAME

fdfs_trac 5913 root  5u  IPv4  30701  0t0  TCP *:22122 (LISTEN)

 

[root@fastdfs-1-1 fdfs]# egrep -v "^#|^$" storage.conf

disabled=false

group_name=group1

bind_addr=

client_bind=true

port=23000

connect_timeout=30

network_timeout=60

heart_beat_interval=30

stat_report_interval=60

base_path=/fdfs/storage/base

max_connections=256

buff_size = 256KB

accept_threads=1

work_threads=4

disk_rw_separated = true

disk_reader_threads = 1

disk_writer_threads = 1

sync_wait_msec=50

sync_interval=0

sync_start_time=00:00

sync_end_time=23:59

write_mark_file_freq=500

store_path_count=1

store_path0=/fdfs/storage/store

subdir_count_per_path=256

tracker_server=192.168.1.128:22122

tracker_server=192.168.1.129:22122

log_level=info

run_by_group=

run_by_user=

allow_hosts=*

file_distribute_path_mode=0

file_distribute_rotate_count=100

fsync_after_written_bytes=0

sync_log_buff_interval=10

sync_binlog_buff_interval=10

sync_stat_file_interval=300

thread_stack_size=512KB

upload_priority=10

if_alias_prefix=

check_file_duplicate=0

file_signature_method=hash

key_namespace=FastDFS

keep_alive=0

use_access_log = false

rotate_access_log = false

access_log_rotate_time=00:00

rotate_error_log = false

error_log_rotate_time=00:00

rotate_access_log_size = 0

rotate_error_log_size = 0

log_file_keep_days = 0

file_sync_skip_invalid_record=false

use_connection_pool = false

connection_pool_max_idle_time = 3600

http.domain_name=

http.server_port=8888

注：两台服务器保持一致即可

[root@fastdfs-1-1 fdfs]# /etc/init.d/fdfs_storaged start

Starting FastDFS storage server:

查看端口

[root@fastdfs-1-1 fdfs]# lsof -i :22122

COMMAND    PID USER   FD   TYPE DEVICE SIZE/OFF NODE NAME

fdfs_trac 3622 root 5u  IPv4  32395  0t0  TCP *:22122 (LISTEN)

fdfs_trac 3622 root 18w  IPv4 100079 0t0  TCP 192.168.1.128:38516->192.168.1.129:22122 (ESTABLISHED)

fdfs_trac 3622 root19u IPv4 87578 0t0 TCP 192.168.1.128:22122->192.168.1.129:40755 (ESTABLISHED)

fdfs_trac 3622 root 22u IPv4 100019 0t0 TCP 192.168.1.128:22122->192.168.1.128:46962 (ESTABLISHED)

fdfs_stor 3640 root 20u  IPv4 100042 0t0 TCP 192.168.1.128:38511->192.168.1.129:22122 (ESTABLISHED)

fdfs_stor 3640 root 21u  IPv4 100043 0t0 TCP 192.168.1.128:46962->192.168.1.128:22122 (ESTABLISHED)

[root@fastdfs-1-1 fdfs]# lsof -i :23000

COMMAND  PID USER   FD   TYPE DEVICE SIZE/OFF NODE NAME

fdfs_stor 3640 root 5u  IPv4 100024 0t0 TCP *:inovaport1 (LISTEN)

fdfs_stor 3640 root 19u  IPv4 100040 0t0 TCP 192.168.1.128:inovaport1->192.168.1.129:42800 (ESTABLISHED)

fdfs_stor 3640 root  23u  IPv4 100076 0t0  TCP 192.168.1.128:50933->192.168.1.129:inovaport1 (ESTABLISHED)

 

05

FastDFS客户端配置

客户端配置文件修改后如下

[root@fastdfs-1-1 ~]# egrep -v "^#|^$" /etc/fdfs/client.conf

connect_timeout=30

network_timeout=60

base_path=/tmp

tracker_server=192.168.1.128:22122

tracker_server=192.168.1.129:22122

log_level=info

use_connection_pool = false

connection_pool_max_idle_time = 3600

load_fdfs_parameters_from_tracker=false

use_storage_id = false

storage_ids_filename = storage_ids.conf

http.tracker_server_port=80

测试上传文件

[root@fastdfs-1-1 ~]# fdfs_upload_file

Usage: fdfs_upload_file [storage_ip:port] [store_path_index]

[root@fastdfs-1-1 ~]# fdfs_upload_file /etc/fdfs/client.conf /etc/passwd

group1/M00/00/00/wKgBgFi9Z2aAXQdkAAAHT15G9sc8427272

返回客户端一个fileid

 

 

[root@fastdfs-1-1 data]# pwd

/fdfs/storage/store/data

默认在数据存储目录下会创建256个目录,然后在每个目录中又创建256个目录,如下图

[root@fastdfs-1-1 00]# cd 00/

[root@fastdfs-1-1 00]# ls

wKgBgFi9Z2aAXQdkAAAHT15G9sc8427272

##刚刚上传的文件

[root@fastdfs-1-1 00]# md5sum wKgBgFi9Z2aAXQdkAAAHT15G9sc8427272

9ced869ca031da653ee8b3b708666a28  wKgBgFi9Z2aAXQdkAAAHT15G9sc8427272

将上传的文件下载下来

[root@fastdfs-1-1 ~]# fdfs_download_file /etc/fdfs/client.conf group1/M00/00/00/wKgBgFi9Z2aAXQdkAAAHT15G9sc8427272          

通过检测MD5值来判断是否是同一文件

[root@fastdfs-1-1 ~]# md5sum wKgBgFi9Z2aAXQdkAAAHT15G9sc8427272

9ced869ca031da653ee8b3b708666a28  wKgBgFi9Z2aAXQdkAAAHT15G9sc8427272

[root@fastdfs-1-1 ~]# md5sum /etc/passwd

9ced869ca031da653ee8b3b708666a28  /etc/passwd

 

06

FastDFS常用命令介绍

fdfs_appender_test ：追加测试

fdfs_crc32           

fdfs_file_info ：查看文件相关信息

fdfs_test ：文件测试

fdfs_upload_appender ：更新追加文件

fdfs_appender_test1 ：追加文件测试

fdfs_delete_file  ：删除文件

fdfs_monitor  ：监控客户端

fdfs_test1   ：测试

fdfs_upload_file ：上传文件

fdfs_append_file ：追加文件（先使用fdfs_upload_appender命令更新文件）

fdfs_download_file ：下载文件

fdfs_storaged   ：存储系统启动命令

fdfs_trackerd   ：追踪系统启动命令

实例操作

查看文件信息

[root@fastdfs-1-1 ~]# fdfs_file_info /etc/fdfs/client.conf group1/M00/00/00/wKgBgFi9Z2aAXQdkAAAHT15G9sc8427272

source storage id: 0

source ip address: 192.168.1.128

file create timestamp: 2017-03-06 08:43:02

file size: 1871

file crc32: 1581708999 (0x5E46F6C7)

删除文件

[root@fastdfs-1-1 ~]# fdfs_delete_file /etc/fdfs/client.conf +文件ID

追加文件内容

[root@fastdfs-1-1 ~]# echo "hello" >test01.txt

[root@fastdfs-1-1 ~]# echo "world" >test02.txt    

[root@fastdfs-1-1 ~]# fdfs_upload_appender /etc/fdfs/client.conf test01.txt

group1/M00/00/00/wKgBgFi9bM6EcC2vAAAAAHcc3SA979.txt

[root@fastdfs-1-1 ~]# fdfs_download_file /etc/fdfs/client.conf group1/M00/00/00/wKgBgFi9bM6EcC2vAAAAAHcc3SA979.txt

[root@fastdfs-1-1 ~]# cat wKgBgFi9bM6EcC2vAAAAAHcc3SA979.txt

hello

[root@fastdfs-1-1 ~]# fdfs_append_file /etc/fdfs/client.conf group1/M00/00/00/wKgBgFi9bM6EcC2vAAAAAHcc3SA979.txt test02.txt

[root@fastdfs-1-1 ~]# fdfs_download_file /etc/fdfs/client.conf group1/M00/00/00/wKgBgFi9bM6EcC2vAAAAAHcc3SA979.txt

[root@fastdfs-1-1 ~]# cat wKgBgFi9bM6EcC2vAAAAAHcc3SA979.txt

hello

world