FastDFS之集群部署
FastDFS是一个开源的轻量级分布式文件系统，由跟踪服务器（tracker server）、存储服务器（storage server）和客户端（client）三个部分组成，主要解决了海量数据存储问题，特别适合以中小文件（建议范围：4KB < file_size <500MB）为载体的在线服务。在生成环境FastDFS一般都是用集群配置，以提高FastDFS的可用性，并发能力。
部署架构：
环境IP地址(关闭所有环境的防火墙)：
Tracker 192.168.18.178
Group 1：
S1：192.168.110.71
S2：192.168.110.91
Group 2：
S3：192.168.100.90
S4：192.168.100.194
注：Tracker可以部署多台，提供负载，这里资源有限，就部署一台。
由于需要安装nginx，每台机器都安装依赖：
yum -y install zlib pcre pcre-devel zlib-devel
所用的安装软件下载：http://download.csdn.net/detail/tianwei7518/8745279
一、安装tracker
1.安装依赖libfastcommon
unzip libfastcommon-master.zip
cd libfastcommon
./make.sh
./make.sh install
2.安装FastDFS
unzip fastdfs.zip
cd fastdfs
./make.sh
./make.sh install
默认安装目录：/usr/bin
将原安装文件夹下的配置文件复制到/etc/fdfs目下：cp ./conf/* /etc/fdfs/
3.配置
编辑配置文件目录下的tracker.conf
一般只需改动以下几个参数即可：
disabled=false #启用配置文件
port=22122 #设置tracker的端口号
base_path=/home/fastdfs #设置tracker的数据文件和日志目录（需预先创建）
http.server_port=8080 #设置http端口号
4.启动
/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf start
二、安装tracker代理nginx
在tracker上安装的nginx主要为了提供http访问的反向代理、负载均衡以及缓存服务。
1.安装nginx
tar -zxvf nginx-1.8.0.tar.gz
tar -zxvf ngx_cache_purge-2.3.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx --add-module=/root/ngx_cache_purge-2.3
make
make install
如果提示错误，可能缺少依赖的软件包，需先安装依赖包，再次运行./configure
nginx以及nginx cache purge插件模块安装完成，安装目录/usr/local/nginx
2.配置nginx
user root;
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
#设置缓存参数
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 300m;
sendfile on;
tcp_nopush on;
proxy_redirect off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 16k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
#设置缓存存储路径、存储方式、分配内存大小、磁盘最大空间、缓存期限
proxy_cache_path /var/cache/nginx/proxy_cache levels=1:2 keys_zone=http-cache:500m max_size=10g inactive=30d;
proxy_temp_path /var/cache/nginx/proxy_cache/tmp;
keepalive_timeout 65;
#设置group服务器
upstream fdfs_group1 {
server 192.168.110.71:8090 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.110.91:8090 weight=1 max_fails=2 fail_timeout=30s;
}
upstream fdfs_group2 {
server 192.168.100.90:8090 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.100.194:8090 weight=1 max_fails=2 fail_timeout=30s;
}
server {
listen 80;
server_name localhost;
charset utf-8;
#access_log /usr/local/nginx/logs/localhost.access.log main;
location /group1/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache http-cache;
proxy_cache_valid 200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group1;
expires 30d;
}
location /group2/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache http-cache;
proxy_cache_valid 200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group2;
expires 30d;
}
#设置清除缓存的访问权限
location ~ /purge(/.*) {
allow 127.0.0.1;
allow 172.16.1.0/24;
deny all;
proxy_cache_purge http-cache $1$is_args$args;
}
}
}创建缓存目录：/var/cache/nginx/proxy_cache/tmp
3.启动/usr/local/nginx/sbin/nginx
三、安装storage
1.安装
参考安装tracker前2步骤。
2.配置
编辑配置文件目录下的storage.conf
只需改动以下几个参数即可：
disabled=false #启用配置文件
group_name=group1 #组名，根据实际情况修改
port=23000 #设置storage的端口号
base_path=/home/fastdfs #设置storage的日志目录（需预先创建）
store_path_count=1 #存储路径个数，需要和store_path个数匹配
store_path0=/home/fastdfs#存储路径
tracker_server=192.168.17.43:22122#tracker服务器的IP地址和端口号
http.server_port=8080 #设置http端口号创建目录mkdir /home/fastdfs
3.运行
/usr/bin/fdfs_storaged /etc/fdfs/storage.conf start
另外：
分别在其他机器上全部安装storage并确认运行正常。注意配置文件中group名参数需要根据实际情况调整：
group1：192.168.110.71，192.168.110.91
group2：192.168.100.90，192.168.100.194
另外每个group中所有storage的端口号必须一致。
四、在storage上安装nginx
在storage上安装的nginx主要为了提供http的访问服务，同时解决group中storage服务器的同步延迟问题。
1.解压fastdfs-nginx-module插件
unzip fastdfs-nginx-module.zip
2.安装nginx
tar -zxvf nginx-1.8.0.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx --add-module=../fastdfs-nginx-module/src
make
make install
安装目录：/usr/local/nginx
若安装报错：[emerg] 13513#0: eventfd() failed (38: Function not implemented)
原因是：编译时带了--with-file-aio模块，这个要linux 2.6.22以后内核才支持．服务器是2.6.18。也可以下载低版本的nginx版本
3.配置
1）配置FastDFS的nginx插件
将FastDFS的nginx插件模块的配置文件copy到FastDFS配置文件目录
fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/
编辑/etc/fdfs配置文件目录下的mod_fastdfs.conf，设置storage信息并保存。
一般只需改动以下几个参数即可：
base_path=/home/fastdfs #保存日志目录
tracker_server=192.168.18.43:22122 #tracker服务器的IP地址以及端口号
storage_server_port=23000#storage服务器的端口号
group_name=group1#当前服务器的group名
url_have_group_name = true #文件url中是否有group名
store_path_count=1 #存储路径个数，需要和store_path个数匹配
store_path0=/home/fastdfs #存储路径
http.need_find_content_type=true#从文件扩展名查找文件类型（nginx时为true）
group_count = 2 #设置组的个数
在末尾增加2个组的具体信息：
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/home/fastdfs
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/home/fastdfs
建立M00至存储目录的符号连接：
ln -s /home/fastdfs/data /home/fastdfs/data/M00
2）配置nginx
vi nginx.conf
user root;
location ~/group[1-3]/M00 {
root /home/fastdfs/data;
ngx_fastdfs_module;
}
4.启动
/usr/local/nginx/sbin/nginx
另外：
分别在其他机器storage上全部安装nginx并确认运行正常。注意配置文件中group名参数需要根据实际情况调整：
group1：192.168.110.71，192.168.110.91
group2：192.168.100.90，192.168.100.194
另外nginx的端口号8090。
至此所有配置完毕。
四、测试
配置/etc/fdfs/client.conf
base_path=/home/fastdfs #日志存放路径
tracker_server=192.168.18.43:22122 #tracker服务器IP地址和端口号
http.tracker_server_port=8080 #tracker服务器的http端口号
通过fdfs_upload_file上传一个文件到FastDFS，程序会自动返回文件的URL，
#fdfs_upload_file /etc/fdfs/client.conf 40-15052PZK5.jpg
group1/M00/00/00/wKhuR1Vmh_2ADmdfAAF1dmVtk4w934.jpg
然后使用浏览器访问，访问正常
http://192.168.18.43/group1/M00/00/00/wKhuR1Vmh_2ADmdfAAF1dmVtk4w934.jpg
注：可以使用fdfs_monitor查看tracker和所有group的运行情况
# fdfs_monitor /etc/fdfs/client.conf
[2015-05-27 20:19:59] DEBUG - base_path=/home/fastdfs, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0
server_count=1, server_index=0
tracker server is 192.168.18.43:22122
group count: 2
Group 1:
group name = group1
disk total space = 45438 MB
disk free space = 33920 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8080
store path count = 1
subdir count per path = 256
current write server index = 1
current trunk file id = 0
Storage 1:
id = 192.168.110.71
ip_addr = 192.168.110.71 (localhost) ACTIVE
http domain =
version = 5.06
join time = 2015-05-27 01:37:04
up time =
total storage = 95217 MB
free storage = 47563 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8080
current_write_path = 0
source storage id =
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 2
total_upload_count = 1
success_upload_count = 1
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 95606
success_upload_bytes = 95606
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 0
success_sync_in_bytes = 0
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 1
success_file_open_count = 1
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 1
success_file_write_count = 1
last_heart_beat_time = 2015-05-27 20:19:49
last_source_update = 2015-05-27 20:14:04
last_sync_update = 1969-12-31 16:00:00
last_synced_timestamp = 1969-12-31 16:00:00
Storage 2:
id = 192.168.110.91
ip_addr = 192.168.110.91 (localhost) ACTIVE
http domain =
version = 5.06
join time = 2015-05-27 19:35:36
up time = 2015-05-27 19:35:36
total storage = 45438 MB
free storage = 33920 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8080
current_write_path = 0
source storage id = 192.168.110.71
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 1
total_upload_count = 0
success_upload_count = 0
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 0
success_upload_bytes = 0
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 95606
success_sync_in_bytes = 95606
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 1
success_file_open_count = 1
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 1
success_file_write_count = 1
last_heart_beat_time = 2015-05-27 20:19:51
last_source_update = 1969-12-31 16:00:00
last_sync_update = 2015-05-27 20:14:07
last_synced_timestamp = 2015-05-27 20:14:05 (-1s delay)
Group 2:
group name = group2
disk total space = 9916 MB
disk free space = 7434 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8080
store path count = 1
subdir count per path = 256
current write server index = 0
current trunk file id = 0
Storage 1:
id = 192.168.100.194
ip_addr = 192.168.100.194 (localhost) ACTIVE
http domain =
version = 5.06
join time = 2015-05-27 20:03:37
up time = 2015-05-27 20:03:37
total storage = 47368 MB
free storage = 37371 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8080
current_write_path = 0
source storage id = 192.168.100.90
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 1
total_upload_count = 0
success_upload_count = 0
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 0
success_upload_bytes = 0
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 0
success_sync_in_bytes = 0
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 0
success_file_open_count = 0
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 0
success_file_write_count = 0
last_heart_beat_time = 2015-05-27 20:19:44
last_source_update = 1969-12-31 16:00:00
last_sync_update = 1969-12-31 16:00:00
last_synced_timestamp = 1969-12-31 16:00:00
Storage 2:
id = 192.168.100.90
ip_addr = 192.168.100.90 (localhost) ACTIVE
http domain =
version = 5.06
join time = 2015-05-27 19:50:27
up time = 2015-05-27 19:50:27
total storage = 9916 MB
free storage = 7434 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8080
current_write_path = 0
source storage id =
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 1
total_upload_count = 0
success_upload_count = 0
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 0
success_upload_bytes = 0
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 0
success_sync_in_bytes = 0
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 0
success_file_open_count = 0
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 0
success_file_write_count = 0
last_heart_beat_time = 2015-05-27 20:19:48
last_source_update = 1969-12-31 16:00:00
last_sync_update = 1969-12-31 16:00:00
last_synced_timestamp = 1969-12-31 16:00:00 1.fastdfs 5.01 + nginx + cache 集群安装配置手册
2.分布式文件系统fastdfs架构剖析及配置优化
3.FastDFS安装所需软件包