分布式文件管理系统FastDFS
FastDFS概述
FastDFS是一个开源的轻量级的分布式文件系统，他对文件进行管理。功能包括：文件存储、文件同步、文件访问（文件上传、文件下载）等。解决了大量的存储和负载均衡的问题。特别适合以文件为载体的在线服务，如相册网站，视频网站等等。
FastDFS为互联网量身定制，充分考虑了冗余备份、负载均衡、线性扩容等机制，并注重高可用、高性能等指标，使用FastDFS很容易搭建一套高性能的文件服务器集群提供文件上传、下载等服务。
FastDFS服务器有两个角色：跟踪器（tracker）和储存节点（storage）。跟踪器主要做调度工作，在访问上起负载均衡的作用。
FastDFS环境搭建
安装Libevent
//解压缩安装包
tar -zxvf libevent-1.4.14b-stable.tar.gz
//进入解压缩目录
cd libevent-1.4.14b-stable
//配置安装路径
./configure --prefix=/usr/local/libevent
//编译
make
//安装
make install
安装FastDFS
//解压缩安装包
tar -zxvf FastDFS_v4.04.tar.gz
//进入解压缩目录
cd FsatDFS
//编辑编译文件make.sh
WITH_HTTPD
WITH_LINUX_SERVICE
//编译
./make.sh C_INCLUDE_PATH=/usr/local/libevent/include LIBRARY_PATH=/usr/local/libevent/lib
//安装
./make.sh install
启动Tracker
//创建文件存储目录/www/fastDFS
mkdir -m 777 -p/www/fastDFS
//修改Tracker配置文件/etc/fdfs/tracker.conf
base_path=/www/fastDFS
http.server_port=8888
#include http.conf
//启动Tracker
/usr/local/bin/fdfs_trackerd /etc/fdfs/tracker.conf
//查看Tracker运行状态
netstat -tnlp
*22122和8888端口LISTEN状态说明启动成功
启动Storage
//修改Stirage配置文件/ect/fdfs/storage.conf
base_path=/www/fastDFS
store_path0=/www/fastDFS
tracker_server=192.168.0.1:22122(根据Tracker的ip地址)
http:disabled=true(关闭http可见，后续使用Nginx)
http.server_port=80
//启动Storage
/usr/local/bin/fdfs_storaged /etc/fdfs/storage.conf
//查看Storage运行状态
netstart -tnlp
*23000端口LISTEN状态说明启动成功
//查看Tracker和Storage连接状态
netstart -tnlpa|grep 22122
使用Client测试上传文件
// 修改Client配置文件 /etc/fdfs/client.conf
base_path=/www/fastDFS
tracker_server=192.168.0.1:22122 （根据Tracker的ip地址）
http.tracker_server_port=8088 （根据Tracker的端口号）
#include http.conf
// 测试上传文件
fdfs_test /etc/fdfs/client.conf upload /tmp/test.txt （根据测试的上传文件路径）
Storage整合Nginx
//Nginx环境搭建
参看Nginx相关博客，之前有介绍。
//解压缩fastdfs-nginx-module
tar –zxvf fastdfs-nginx-module_v1.13.tar.gz
//进入Nginx解压缩目录
cd Nginx-x.x.x
//Nginx添加fastdfs-nginx-module模块
./configure --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module --add-module=/usr/local/fastdfs-nginx-module/src
//编译
make
//安装
make install
// 复制fastdfs-nginx-module配置文件
cp /usr/local/fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/
// 修改fastdfs-nginx-module配置文件
connect_timeout=20
base_path=/www/fastDFS/
tracker_server=192.168.0.1:22122 （根据Tracker的ip地址）
url_have_group_name = true （设置文件的URL带上Group名称）
store_path0=/www/fastDFS
// 创建Storage存储路径连接
ln -s /www/fastDFS/data /www/fastDFS/data/M00
// 编辑Nginx配置文件 /usr/local/nginx/conf/nginx.conf
Server{
……
location /group1/M00/ { alias /www/fastDFS/data/; ngx_fastdfs_module; }
……
}
// 关闭后重新启动Nginx
/usr/local/nginx/sbin/nginx -t
/usr/local/nginx/sbin/nginx -s stop
/usr/local/nginx/sbin/nginx
SSM整合FastDFS
Pom添加依赖
FastDFS Client资源文件：fastdfs-client.properties
| connect_timeout = 2 network_timeout = 30 charset = UTF-8 http.tracker_http_port = 8888 http.anti_steal_token = no http.secret_key = FastDFS1234567890 tracker_server = 172.31.79.54:22122 storage_server = 172.31.79.58:23000 |
FastDFS Client配置接口：FileManagerConfig
| package com.bwf.ssm.common; import java.io.Serializable; public interface FileManagerConfig extends Serializable { public static final String FILE_DEFAULT_WIDTH = "120"; public static final String FILE_DEFAULT_HEIGHT = "120"; public static final String FILE_DEFAULT_AUTHOR = "BWF"; public static final String PROTOCOL = "http://"; public static final String SEPARATOR = "/"; public static final String TRACKER_NGNIX_PORT = "80"; public static final String CLIENT_CONFIG_FILE = "fastdfs-client.properties"; } |
FastDFS文件实体类：FastDFSFile
| package com.bwf.ssm.common; public class FastDFSFile implements FileManagerConfig { private static final long serialVersionUID = -996760121932438618L; private String name; private byte[] content; private String ext; private String height = FILE_DEFAULT_HEIGHT; private String width = FILE_DEFAULT_WIDTH; private String author = FILE_DEFAULT_AUTHOR; public FastDFSFile(String name, byte[] content, String ext, String height, String width, String author) { super(); this.name = name; this.content = content; this.ext = ext; this.height = height; this.width = width; this.author = author; } public FastDFSFile(String name, byte[] content, String ext) { super(); this.name = name; this.content = content; this.ext = ext; } public byte[] getContent() { return content; } public void setContent(byte[] content) { this.content = content; } public String getExt() { return ext; } public void setExt(String ext) { this.ext = ext; } public String getHeight() { return height; } public void setHeight(String height) { this.height = height; } public String getWidth() { return width; } public void setWidth(String width) { this.width = width; } public String getAuthor() { return author; } public void setAuthor(String author) { this.author = author; } public String getName() { return name; } public void setName(String name) { this.name = name; } } |
FastDFS文件管理类：FileManager
| package com.bwf.ssm.common; import java.io.File; import java.io.IOException; import org.apache.log4j.Logger; import org.csource.common.NameValuePair; import org.csource.fastdfs.ClientGlobal; import org.csource.fastdfs.FileInfo; import org.csource.fastdfs.ServerInfo; import org.csource.fastdfs.StorageClient; import org.csource.fastdfs.StorageServer; import org.csource.fastdfs.TrackerClient; import org.csource.fastdfs.TrackerServer; public class FileManager implements FileManagerConfig { private static final long serialVersionUID = 1L; private static Logger logger = Logger.getLogger(FileManager.class); private static TrackerClient trackerClient; private static TrackerServer trackerServer; private static StorageServer storageServer; private static StorageClient storageClient; static { try { String classPath = new File(FileManager.class.getResource("/").getFile()).getCanonicalPath(); String fdfsClientConfigFilePath = classPath + File.separator + CLIENT_CONFIG_FILE; System.out.println("Fast DFS configuration file path:" + fdfsClientConfigFilePath); ClientGlobal.init(fdfsClientConfigFilePath); trackerClient = new TrackerClient(); trackerServer = trackerClient.getConnection(); storageClient = new StorageClient(trackerServer, storageServer); } catch (Exception e) { e.printStackTrace(); } } public static String[] upload(FastDFSFile file) { NameValuePair[] meta_list = new NameValuePair[3]; meta_list[0] = new NameValuePair("width", file.getWidth()); meta_list[1] = new NameValuePair("heigth", file.getHeight()); meta_list[2] = new NameValuePair("author", file.getAuthor()); String[] uploadResults = null; try { uploadResults = storageClient.upload_file(file.getContent(), file.getExt(), meta_list); } catch (Exception e) { e.printStackTrace(); } if (uploadResults == null) { System.out.println("upload file fail, error code: " + storageClient.getErrorCode()); } String groupName = uploadResults[0]; String remoteFileName = uploadResults[1]; String fileAbsolutePath = "" // + PROTOCOL // + storageServer.getInetSocketAddress().getHostName() // + ":" // + TRACKER_NGNIX_PORT // + SEPARATOR + groupName + SEPARATOR + remoteFileName; System.out.println("upload file successfully!!! " +"group_name: " + groupName + ", remoteFileName:" + " " + remoteFileName); return uploadResults; } public static FileInfo getFile(String groupName, String remoteFileName) { try { return storageClient.get_file_info(groupName, remoteFileName); } catch (Exception e) { e.printStackTrace(); } return null; } public static byte[] downloadFile(String groupName, String remoteFileName) throws Exception { return storageClient.download_file(groupName, remoteFileName); } public static void deleteFile(String groupName, String remoteFileName) throws Exception { storageClient.delete_file(groupName, remoteFileName); } public static StorageServer[] getStoreStorages(String groupName) throws IOException { return trackerClient.getStoreStorages(trackerServer, groupName); } public static ServerInfo[] getFetchStorages(String groupName, String remoteFileName) throws IOException { return trackerClient.getFetchStorages(trackerServer, groupName, remoteFileName); } } |
上传文件
| @RequestMapping("/upload") public String upload( @RequestParam("photoName") String photoName, @RequestParam("face") MultipartFile file) throws Exception { // 获取上传文件原文件名和扩展名 String filename = file.getOriginalFilename().substring(0, file.getOriginalFilename().lastIndexOf(".")-1); String extname = file.getOriginalFilename().substring( file.getOriginalFilename().lastIndexOf(".")+1 ); // 实例化FastDFS文件实体对象 FastDFSFile f = new FastDFSFile(filename, file.getBytes(), extname ); // 传送文件到FastDFS的Storage，返回Stroage的Group和文件路径 String[] result = FileManager.upload( f); System.out.println(" ====> group : " + result[0]+" , remote: "+ result[1]); // 根据上传的文件实例化Photo实体对象 Photo photo = new Photo(); photo.setPhotoName(photoName); photo.setStorageGroup(result[0]); photo.setStorageUrl(result[1]); // 添加到数据库中 photoservice.add(photo); return "redirect:/index/index"; } |
下载文件
| @RequestMapping("/download/{photoId}") public ResponseEntity @PathVariable("photoId") Integer photoId) throws Exception{ // 从数据库中查询要下载的文件对象 Photo photo = photoservice.getPhotoById(photoId); // 从FastDFS的Storage中，根据Group和文件路径，获取文件 byte[] data = FileManager.downloadFile(photo.getStorageGroup(), photo.getStorageUrl()); // 设置响应头信息 HttpHeaders head = new HttpHeaders(); // 设置下载文件默认名称 head.setContentDispositionFormData("attachment", photo.getStorageUrl().substring(photo.getStorageUrl().lastIndexOf("/"))); // 设置客户端浏览器接受响应数据的类型 head.setContentType(MediaType.APPLICATION_OCTET_STREAM); // 打开文件，读取文件数据，响应给客户端浏览器 return new ResponseEntity } |