FastDFS高可用集群架构配置搭建及使用

一,概述

FastDFS 是一个开源的高性能分布式文件系统(DFS)。 它的主要功能包括:文件存储,文件同步和文件访问,以及高容量和负载平衡。

FastDFS 系统有三个角色:跟踪服务器(Tracker Server)、存储服务器(Storage Server)和客户端(Client)。

Tracker Server: 跟踪服务器,主要做调度工作,起到均衡的作用;负责管理所有的storage server和group,每个storage在启动后会连接 Tracker,告知自己所属 group 等信息,并保持周期性心跳。多个Tracker之间是对等关系,不存在单点故障。

Storage Server: 存储服务器,主要提供容量和备份服务;以 group 为单位,每个 group 内可以有多台 storage server,组内的storage server上的数据互为备份。

Client:客户端,上传下载数据的服务器

模块之间的主要关系如下:

下图是实现统一的对外下载访问入口的高可用架构,其中所有的Nginx只做下载用途,上传通过tracker进行上传。

二、环境准备工作

三、安装过程

1, tracker

#安装编译环境

$ yum groups install Development Tools -y

$ yum install perl -y

$ mkdir /opt/fastdfs

$ mkdir /data/fastdfs

$ cd /opt/fastdfs #为下一步下载源码做准备

安装libfastcommon

#下载文件

$ wget https://github.com/happyfish100/libfastcommon/archive/V1.0.39.tar.gz

$ tar -zxvf libfastcommon-1.0.39.tar.gz

$ cd libfastcommon-1.0.39/

$ ./make.sh

$ ./make.sh install

安装FastDFS

wget https://github.com/happyfish100/fastdfs/archive/V5.11.tar.gz

$ tar -zxvf fastdfs-5.11.tar.gz

$ cd fastdfs-5.11/

$ ./make.sh

$ ./make.sh install

#配置文件准备

$ cp /etc/fdfs/tracker.conf.sample /etc/fdfs/tracker.conf #tracker节点 10.250.112.141/142

$ cp /etc/fdfs/storage.conf.sample /etc/fdfs/storage.conf #storage节点 10.250.112.143/144

$ cp /etc/fdfs/client.conf.sample /etc/fdfs/client.conf #客户端文件,测试用。10.250.112.141/142/143/144

$ cp /opt/fastdfs/fastdfs-5.11/conf/http.conf /etc/fdfs/ #供nginx访问使用  10.250.112.143/144

$ cp /opt/fastdfs/fastdfs-5.11/conf/mime.types /etc/fdfs/ #供nginx访问使用  10.250.112.143/144

tracker server配置:

$ vim /etc/fdfs/tracker.conf

#需要修改的内容如下

port=22122 # tracker服务器端口(默认22122,一般不修改)

base_path=/data/fastdfs # 存储日志和数据的根目录

#编辑启动文件

$ vim /usr/lib/systemd/system/fastdfs-tracker.service

[Unit]

Description=The FastDFS File server

After=network.target remote-fs.target nss-lookup.target

[Service]

Type=forking

ExecStart=/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf start

ExecStop=/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf stop

ExecRestart=/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf restart

[Install]

WantedBy=multi-user.target

$ systemctl daemon-reload

$ systemctl enable fastdfs-tracker.service  #不一定成功,不过没关系

$ systemctl start fastdfs-tracker.service  #不一定成功,不过没关系

$ netstat -tulnp #查看服务是否启动,端口是否打开

2, Storage

storage server配置

$ vim /etc/fdfs/storage.conf

#需要修改的内容如下

port=23000 # storage服务端口(默认23000,一般不修改)

base_path=/data/fastdfs # 数据和日志文件存储根目录

store_path0=/data/fastdfs # 第一个存储目录

tracker_server=10.250.112.141:22122 # tracker服务器IP和端口

tracker_server=10.250.112.142:22122 # tracker服务器IP和端口

http.server_port=8888 # http访问文件的端口(默认8888,看情况修改,和nginx中保持一致)

#编辑启动文件

$ vim /usr/lib/systemd/system/fastdfs-storage.service

[Unit]

Description=The FastDFS File server

After=network.target remote-fs.target nss-lookup.target

[Service]

Type=forking

ExecStart=/usr/bin/fdfs_storaged /etc/fdfs/storage.conf start

ExecStop=/usr/bin/fdfs_storaged /etc/fdfs/storage.conf stop

ExecRestart=/usr/bin/fdfs_storaged /etc/fdfs/storage.conf restart

[Install]

WantedBy=multi-user.target

$ systemctl daemon-reload

$ systemctl enable fastdfs-storage.service

$ systemctl start fastdfs-storage.service

$ netstat -tulnp #查看服务是否启动,端口是否打开

#查看集群状态

$ fdfs_monitor /etc/fdfs/storage.conf list

Client配置

$ vim /etc/fdfs/client.conf

#需要修改的内容如下

base_path=/data/fastdfs

tracker_server=10.250.112.141:22122 # tracker服务器IP和端口

tracker_server=10.250.112.142:22122 # tracker服务器IP和端口

#保存后测试,返回ID表示成功 如:group1/M00/00/00/xx.tar.gz

$ fdfs_upload_file /etc/fdfs/client.conf testfile

3,安装nginx和fastdfs-nginx-module

在10.250.112.143和10.250.112.144上

下载nginx module

$ wget https://github.com/happyfish100/fastdfs-nginx-module/archive/V1.20.tar.gz

$ cp /opt/fastdfs/fastdfs-nginx-module-1.20/src/mod_fastdfs.conf /etc/fdfs

安装nginx

wget http://nginx.org/download/nginx-1.12.2.tar.gz

$ tar -zxvf nginx-1.12.2.tar.gz

$ cd nginx-1.12.2/

$ ./configure --prefix=/usr/local/nginx --sbin-path=/usr/local/nginx --modules-path=/usr/local/nginx/modules --conf-path=/usr/local/nginx/conf/nginx.conf --error-log-path=/usr/local/nginx/logs/error.log --http-log-path=/usr/local/nginx/logs/access.log --http-client-body-temp-path=/var/lib/nginx/tmp/client_body --http-proxy-temp-path=/var/lib/nginx/tmp/proxy --http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi --http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi --http-scgi-temp-path=/var/lib/nginx/tmp/scgi --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx --user=nginx --group=nginx --with-file-aio --with-http_auth_request_module --with-http_ssl_module --with-http_v2_module --with-http_realip_module --with-http_addition_module --with-http_xslt_module=dynamic --with-http_image_filter_module=dynamic --with-http_geoip_module=dynamic --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_degradation_module --with-http_slice_module --with-http_stub_status_module --with-http_perl_module=dynamic --with-mail=dynamic --with-mail_ssl_module --with-pcre --with-pcre-jit --with-stream=dynamic --with-stream_ssl_module --with-google_perftools_module --with-debug --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic' --with-ld-opt='-Wl,-z,relro -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -Wl,-E' --add-module=/opt/fastdfs/fastdfs-nginx-module-1.20/src/

$ make

$ cp objs/nginx /usr/sbin/nginx

# 查看版本及编译参数

$ nginx -v

4,配置nginx

在10.250.112.143和10.250.112.144上

$ vim /etc/fdfs/mod_fastdfs.conf

#需要修改的内容如下

tracker_server=192.168.0.1:22122 # tracker服务器IP和端口

tracker_server=192.168.0.2:22122 # tracker服务器IP和端口

url_have_group_name=true

base_path=/data/fastdfs

store_path0=/data/fastdfs

$ vim /etc/nginx/nginx.conf

将原来内容去掉,改为下面的内容

worker_processes  1;

events {

worker_connections  1024;

}

http {

include       mime.types;

default_type  application/octet-stream;

sendfile        on;

#keepalive_timeout  0;

keepalive_timeout  65;

server {

listen       80;

server_name  localhost;

location / {

root   html;

index  index.html index.htm;

}

error_page   500 502 503 504  /50x.html;

location = /50x.html {

root   html;

}

}

server {

listen 8888; ## 该端口为storage.conf中的http.server_port相同

server_name localhost;

location ~/group[0-9]/ {

root /data/fastdfs;

ngx_fastdfs_module;

}

error_page 500 502 503 504 /50x.html;

location = /50x.html {

root html;

}

}

}

注意:1,启动nginx, 用whereis nginx定位nginx的位置,进到sbin目录,使用 ./nginx 启动。

           2,将用到的端口到防火墙中打开,命令如下

vi /etc/sysconfig/iptables

添加要开放的端口,如8888,9270,22122等用到的端口

service iptables restart 

注:

在nginx的构建中会遇到不少的报错,具体如下:

./configure: error: the Google perftools module requires the Google perftools library. You can either do not enable the module or install the library.

          解决方法如下:

$ yum install gperftools -y

/configure: error: the HTTP rewrite module requires the PCRE library.

解决方法如下:

$ yum -y install pcre pcre-devel

./configure: error: the HTTP cache module requires md5 functions from OpenSSL library. You can either disable the module by using --without-http-cache option, or install the OpenSSL library into the system, or build the OpenSSL library statically from the source with nginx by using --with-http_ssl_module --with-openssl= options.

解决方法如下:

$ yum -y install openssl openssl-devel

./configure: error: the HTTP gzip module requires the zlib library. You can either disable the module by using –without-http_gzip_module option, or install the zlib library into the system, or build the zlib library statically from the source with nginx by using –with-zlib= option.

解决方法如下:

$ yum install -y zlib-devel

./configure: error: the HTTP XSLT module requires the libxml2/libxslt libraries. You can either do not enable the module or install the libraries.

解决方法如下:

$ yum -y install libxml2 libxml2-dev

$ yum -y install libxslt-devel

./configure: error: the HTTP image filter module requires the GD library. You can either do not enable the module or install the libraries.

解决方法如下:

$ yum -y install gd-devel

./configure: error: perl module ExtUtils::Embed is required

解决方法如下:

$ yum -y install perl-devel perl-ExtUtils-Embed

./configure: error: the GeoIP module requires the GeoIP library. You can either do not enable the module or install the library.

解决方法如下:

$ yum -y install GeoIP GeoIP-devel GeoIP-data

在make过程中会出现一个报错:/usr/include/fastdfs/fdfs_define.h:15:27: fatal error: common_define.h: No such file or directory

解决方法如下:

修改fastdfs-nginx-module-1.20/src/config文件,修改如下:

ngx_module_incs="/usr/include/fastdfs /usr/include/fastcommon/"

CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/include/fastcommon/"

然后重新./configure && make,就可以了

5、配置文件访问的负载均衡和高可用

在10.250.112.145和10.250.112.146上安装nginx

wget http://nginx.org/download/nginx-1.12.2.tar.gz

$ tar -zxvf nginx-1.12.2.tar.gz

$ cd nginx-1.12.2/

$ ./configure

$ make

$ make install

安装keepalived

yum install -y nginx keepalived

nginx.conf配置文件如下

worker_processes  1;

events {

worker_connections  1024;

}

http {

include       mime.types;

default_type  application/octet-stream;

sendfile        on;

#keepalive_timeout  0;

keepalive_timeout  65;

upstream fdfs_group {

server 10.250.112.143:8888 weight=1 max_fails=2 fail_timeout=30s;

server 10.250.112.144:8888 weight=1 max_fails=2 fail_timeout=30s;

}

server {

listen 80;

server_name localhost;

location ~/group[0-9]/{

proxy_next_upstream http_502 http_504 error timeout invalid_header;

proxy_pass http://fdfs_group;

expires 30d;

}

}

}

10.250.112.145 上keepalived配置文件如下:

global_defs {

  router_id LVS_DEVEL

}

vrrp_script chk_ngx {

    script "/etc/keepalived/check_nginx.sh"

    interval 2

    weight -5

    fall 3 

    rise 2

}

vrrp_instance VI_1 {

    interface eno16777984

    state MASTER

    priority 100

    virtual_router_id 11

    advert_int 1

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    unicast_src_ip 10.250.112.145

    unicast_peer {

        10.250.112.146

    }

    virtual_ipaddress {

        10.250.112.145

    }

    track_script {

      chk_ngx

    }

    notify_master "/etc/keepalived/notify.sh master" 

    notify_backup "/etc/keepalived/notify.sh backup" 

    notify_fault "/etc/keepalived/notify.sh fault"

}

新建一个check_nginx.sh文件,内容如下

#!/bin/bash

counter=$(ps -C nginx --no-heading|wc -l)

if [ "${counter}" = "0" ]; then

    exit 1

else

    exit 0

fi

到这里,所有的安装都完成了

在10.250.112.143上上传的图片就可以通过下面的路径访问了

http://10.250.112.143:8888/group1/M00/00/00/CvpwkF2BlAWAPEXTAAqPeqUITvA818.png

http://10.250.112.146/group1/M00/00/00/CvpwkF2BlAWAPEXTAAqPeqUITvA818.png

后面一种是做过负载均衡的。

四,代码层面使用FastDFS

建立maven工程,引入


net.oschina.zcx7878

fastdfs-client-java

1.27.0.0

上传文件

@Override

public String upload(String fileName) {

String filePath = getFilePrefix(finalFilePathPrefix);

try {

//            读取本地conf文件配置时

ClientGlobal.init("fdfs_client.conf");

//            读取apollo配置时

//            ClientGlobal.setG_connect_timeout(connectTimeout);

//            ClientGlobal.setG_network_timeout(networkTimeout);

//            ClientGlobal.setG_charset(charset);

//            ClientGlobal.setG_anti_steal_token(httpAntiStealToken);

//            ClientGlobal.setG_secret_key(httpSecretKey);

//            ClientGlobal.setG_tracker_http_port(httpTrackerPort);

//            String[] trackerServers = trackerServerIps.split("\\,");

//            InetSocketAddress[] tracker_servers = new InetSocketAddress[trackerServers.length];

//            for (int i = 0; i < trackerServers.length; i++) {

//              tracker_servers[i] = new InetSocketAddress(trackerServers[i].trim(), trackerServerPort);

//            }

//            TrackerGroup trackerGroup = new TrackerGroup(tracker_servers);

//            ClientGlobal.setG_tracker_group(trackerGroup);

//获取client

TrackerClient tracker = new TrackerClient();

TrackerServer trackerServer = tracker.getConnection();

StorageServer storageServer = null;

StorageClient client = new StorageClient(trackerServer, storageServer);

//上传文件

NameValuePair[] metaList = new NameValuePair[1];

metaList[0] = new NameValuePair("fileName", fileName);

String results[] = client.upload_file(fileName, null, metaList);

filePath = filePath + results[0]+"/"+results[1];

} catch (IOException e) {

e.printStackTrace();

} catch (MyException e) {

e.printStackTrace();

}

return filePath;

}

下载文件

@Override

public String download(String url, String localName) {

if (StringUtil.isBlank(url)) {

return "Fail";

}

String fileName = url.split("/"+groupName+"/")[1];

try {

ClientGlobal.setG_connect_timeout(connectTimeout);

ClientGlobal.setG_network_timeout(networkTimeout);

ClientGlobal.setG_charset(charset);

ClientGlobal.setG_anti_steal_token(httpAntiStealToken);

ClientGlobal.setG_secret_key(httpSecretKey);

ClientGlobal.setG_tracker_http_port(httpTrackerPort);

String trackerServers[] = trackerServerIps.split("\\,");

InetSocketAddress[] tracker_servers = new InetSocketAddress[trackerServers.length];

for (int i = 0; i < trackerServers.length; i++) {

tracker_servers[i] = new InetSocketAddress(trackerServers[i].trim(), trackerServerPort);

}

TrackerGroup trackerGroup = new TrackerGroup(tracker_servers);

ClientGlobal.setG_tracker_group(trackerGroup);

//获取client

TrackerClient tracker = new TrackerClient();

TrackerServer trackerServer = tracker.getConnection();

StorageServer storageServer = null;

StorageClient client = new StorageClient(trackerServer, storageServer);

//0表示下载成功

int count = client.download_file(groupName, fileName,localName);

System.out.println(count);

} catch (IOException | MyException e) {

e.printStackTrace();

}

return "success";

}

fdfs_client.conf 内容如下:

connect_timeout = 2

network_timeout = 30

charset = UTF-8

http.tracker_http_port = 8080

http.anti_steal_token = no

http.secret_key = FastDFS1234567890

tracker_server = 10.250.112.141:22122

tracker_server = 10.250.112.142:22122


码字不易,如果觉得有帮助,一定要给我点赞哟~~

不然信不信我砸了你家灯,半夜偷亲你 ( ̄ε  ̄) !!!

你可能感兴趣的:(FastDFS高可用集群架构配置搭建及使用)