java web 之springboot教程之二十九----FastDFS集成

FastDFS是一个开源的轻量级分布式文件系统，由跟踪服务器（tracker server）、存储服务器（storage server）和客户端（client）三个部分组成，主要解决了海量数据存储问题，特别适合以中小文件（建议范围：4KB < file_size <500MB）为载体的在线服务。

参考：

https://blog.csdn.net/u013378306/article/details/74852355

https://baike.baidu.com/item/fastdfs/5609710?fr=aladdin

 

ubuntu16.04安装fastdfs：

astdfs github地址： https://github.com/happyfish100/

.FastDFS上传原理

 - storage定时向tracker上传状态信息

 - client上传连接请求到tracker

 - tracker查询可用的storage，并返回storage的ip和端口

 - 上传文件fIlecontent和meta data

 - 生成file_id，将上传的内容写入磁盘，并返回file_id路径和文件名

安装之前先安装所需要的库：

libevent

 http://libevent.org/ 下载 最新稳定版

libfastcommon 

git clone  https://github.com/happyfish100/libfastcommon.git

fastdfs 

https://zh.osdn.net/projects/sfnet_fastdfs/releases/

一.安装libevent

  解压；tar  -zxvf  libevent-2.0.22-stable.tar.gz

  进入目录；

  ./configure --prefix=/usr         配置目录

  make 

  make install

  验证   ls -al /usr/lib |grep libevent

二.安装libfastcommon

  unzip libfastcommon-master.zip

  cd  libfastcommon-master 

  ./make.sh

  ./make.sh install

三.安装fastdfs

  解压 tar  -zxvf  FastDFS.tar.gz

  cd FastDFS

  ./make.sh

  ./make.sh install

四.配置 tracker.conf

  cd /etc/fdfs/

  cp tracker.conf.sample tracker.conf

  vi tracker.conf

  base_path=/data/fastdfs/tracker #设置 tracker 的数据文件和日志目录（需预先创建） mkdir -p      /home/fastdfs/tracker 

  store_group=group1   #修改组名

五. 配置 storage.conf

  cp storage.conf.sample storage.conf

  vi storage.conf

  group_name=group1                                          #组名，根据实际情况修改

  base_path=/home/fastdfs/storage          #设置 storage 的日志目录（需预先创建）mkdir -p  /home/fastdfs/storage 

  store_path_count=1                       #存储路径个数，需要和 store_path 个数匹配

  store_path0=/home/fastdfs/storage/datas          #存储路径（需预先创建）mkdir -p  /home/fastdfs/storage/datas

  tracker_server=10.10.10.81:22122                     #tracker 服务器的 IP 地址和端口号

六.配置 client.conf 

  cp client.conf.sample client.conf

  vi client.conf

  base_path=/data/fastdfs/fdfs_client　　　　　  　# 日志路径

  tracker_server=172.16.1.40:22122　　　　　　   # 追踪服务器的IP，有多个服务器可以另一行

启动

启动 fdfs_trackerd /etc/fdfs/tracker.conf

启动 fdfs_storaged /etc/fdfs/storage.conf

停止

直接kill即可让server进程正常退出，可以使用killall命令，例如： 

killall fdfs_trackerd 

killall fdfs_storaged 

springboot整合开始：

1，导入pom.xml

   4.0.0
   
      org.springframework.boot
      spring-boot-starter-parent
      2.1.4.RELEASE
       
   
   com.example
   fastdfs
   0.0.1-SNAPSHOT
   fastdfs
   Demo project for Spring Boot

   
      1.8
   

   
        
        
            net.oschina.zcx7878
            fastdfs-client-java
            1.27.0.0
        
        
            org.springframework.boot
            spring-boot-devtools
            true
        

        
         org.springframework.boot
         spring-boot-starter-thymeleaf
      
      
         org.springframework.boot
         spring-boot-starter-web
      

      
         org.projectlombok
         lombok
         true
      
      
         org.springframework.boot
         spring-boot-starter-test
         test
      
   

   
      
         
            org.springframework.boot
            spring-boot-maven-plugin
         
      
   

2，resources下面的配置

application.properties

#http://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#common-application-properties
#search multipart
spring.servlet.multipart.max-file-size=10MB
spring.servlet.multipart.max-request-size=10MB

fdfs_client.conf

connect_timeout = 60
network_timeout = 60
charset = UTF-8
http.tracker_http_port = 8080
http.anti_steal_token = no
http.secret_key = 123456

tracker_server = 你的IP:22122
tracker_server = 你的IP:22122

3，templates：

Spring Boot file upload example


    


    

Spring Boot - Upload Status


    




    

4，controller

package com.example.fastdfs.controller;

import com.example.fastdfs.fastdfs.FastDFSClient;
import com.example.fastdfs.fastdfs.FastDFSFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.support.RedirectAttributes;

import java.io.IOException;
import java.io.InputStream;

@Controller
public class UploadController {
    private static Logger logger = LoggerFactory.getLogger(UploadController.class);

    @GetMapping("/")
    public String index() {
        return "upload";
    }

    @PostMapping("/upload") //new annotation since 4.3
    public String singleFileUpload(@RequestParam("file") MultipartFile file,
                                   RedirectAttributes redirectAttributes) {
        if (file.isEmpty()) {
            redirectAttributes.addFlashAttribute("message", "Please select a file to upload");
            return "redirect:uploadStatus";
        }
        try {
            // Get the file and save it somewhere
            String path=saveFile(file);
            redirectAttributes.addFlashAttribute("message",
                    "You successfully uploaded '" + file.getOriginalFilename() + "'");
            redirectAttributes.addFlashAttribute("path",
                    "file path url '" + path + "'");
        } catch (Exception e) {
            logger.error("upload file failed",e);
        }
        return "redirect:/uploadStatus";
    }

    @GetMapping("/uploadStatus")
    public String uploadStatus() {
        return "uploadStatus";
    }

    /**
     * @param multipartFile
     * @return
     * @throws IOException
     */
    public String saveFile(MultipartFile multipartFile) throws IOException {
        String[] fileAbsolutePath={};
        String fileName=multipartFile.getOriginalFilename();
        String ext = fileName.substring(fileName.lastIndexOf(".") + 1);
        byte[] file_buff = null;
        InputStream inputStream=multipartFile.getInputStream();
        if(inputStream!=null){
            int len1 = inputStream.available();
            file_buff = new byte[len1];
            inputStream.read(file_buff);
        }
        inputStream.close();
        FastDFSFile file = new FastDFSFile(fileName, file_buff, ext);
        try {
            fileAbsolutePath = FastDFSClient.upload(file);  //upload to fastdfs
        } catch (Exception e) {
            logger.error("upload file Exception!",e);
        }
        if (fileAbsolutePath==null) {
            logger.error("upload file failed,please upload again!");
        }
        String path=FastDFSClient.getTrackerUrl()+fileAbsolutePath[0]+ "/"+fileAbsolutePath[1];
        return path;
    }
}

异常捕获

package com.example.fastdfs.controller;

import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.multipart.MultipartException;
import org.springframework.web.servlet.mvc.support.RedirectAttributes;

@ControllerAdvice
public class GlobalExceptionHandler {

    //https://jira.spring.io/browse/SPR-14651
    //4.3.5 supports RedirectAttributes redirectAttributes
    @ExceptionHandler(MultipartException.class)
    public String handleError1(MultipartException e, RedirectAttributes redirectAttributes) {
        redirectAttributes.addFlashAttribute("message", e.getCause().getMessage());
        return "redirect:/uploadStatus";
    }
}

5，fastdfs操作类：

package com.example.fastdfs.fastdfs;

import org.csource.common.NameValuePair;
import org.csource.fastdfs.*;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.ClassPathResource;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;

public class FastDFSClient {
   private static org.slf4j.Logger logger = LoggerFactory.getLogger(FastDFSClient.class);

   static {
      try {
         String filePath = new ClassPathResource("fdfs_client.conf").getFile().getAbsolutePath();;
         ClientGlobal.init(filePath);
      } catch (Exception e) {
         logger.error("FastDFS Client Init Fail!",e);
      }
   }

   public static String[] upload(FastDFSFile file) {
      logger.info("File Name: " + file.getName() + "File Length:" + file.getContent().length);

      NameValuePair[] meta_list = new NameValuePair[1];
      meta_list[0] = new NameValuePair("author", file.getAuthor());

      long startTime = System.currentTimeMillis();
      String[] uploadResults = null;
      StorageClient storageClient=null;
      try {
         storageClient = getTrackerClient();
         uploadResults = storageClient.upload_file(file.getContent(), file.getExt(), meta_list);
      } catch (IOException e) {
         logger.error("IO Exception when uploadind the file:" + file.getName(), e);
      } catch (Exception e) {
         logger.error("Non IO Exception when uploadind the file:" + file.getName(), e);
      }
      logger.info("upload_file time used:" + (System.currentTimeMillis() - startTime) + " ms");

      if (uploadResults == null && storageClient!=null) {
         logger.error("upload file fail, error code:" + storageClient.getErrorCode());
      }
      String groupName = uploadResults[0];
      String remoteFileName = uploadResults[1];

      logger.info("upload file successfully!!!" + "group_name:" + groupName + ", remoteFileName:" + " " + remoteFileName);
      return uploadResults;
   }

   public static FileInfo getFile(String groupName, String remoteFileName) {
      try {
            StorageClient storageClient = getTrackerClient();
         return storageClient.get_file_info(groupName, remoteFileName);
      } catch (IOException e) {
         logger.error("IO Exception: Get File from Fast DFS failed", e);
      } catch (Exception e) {
         logger.error("Non IO Exception: Get File from Fast DFS failed", e);
      }
      return null;
   }

   public static InputStream downFile(String groupName, String remoteFileName) {
      try {
         StorageClient storageClient = getTrackerClient();
         byte[] fileByte = storageClient.download_file(groupName, remoteFileName);
         InputStream ins = new ByteArrayInputStream(fileByte);
         return ins;
      } catch (IOException e) {
         logger.error("IO Exception: Get File from Fast DFS failed", e);
      } catch (Exception e) {
         logger.error("Non IO Exception: Get File from Fast DFS failed", e);
      }
      return null;
   }

   public static void deleteFile(String groupName, String remoteFileName)
         throws Exception {
      StorageClient storageClient = getTrackerClient();
      int i = storageClient.delete_file(groupName, remoteFileName);
      logger.info("delete file successfully!!!" + i);
   }

   public static StorageServer[] getStoreStorages(String groupName)
         throws IOException {
      TrackerClient trackerClient = new TrackerClient();
      TrackerServer trackerServer = trackerClient.getConnection();
      return trackerClient.getStoreStorages(trackerServer, groupName);
   }

   public static ServerInfo[] getFetchStorages(String groupName,
                                    String remoteFileName) throws IOException {
      TrackerClient trackerClient = new TrackerClient();
      TrackerServer trackerServer = trackerClient.getConnection();
      return trackerClient.getFetchStorages(trackerServer, groupName, remoteFileName);
   }

   public static String getTrackerUrl() throws IOException {
      return "http://"+getTrackerServer().getInetSocketAddress().getHostString()+":"+ClientGlobal.getG_tracker_http_port()+"/";
   }

   private static StorageClient getTrackerClient() throws IOException {
      TrackerServer trackerServer = getTrackerServer();
      StorageClient storageClient = new StorageClient(trackerServer, null);
      return  storageClient;
   }

   private static TrackerServer getTrackerServer() throws IOException {
      TrackerClient trackerClient = new TrackerClient();
      TrackerServer trackerServer = trackerClient.getConnection();
      return  trackerServer;
   }
}

package com.example.fastdfs.fastdfs;

public class FastDFSFile {
   private String name;

   private byte[] content;

   private String ext;

   private String md5;

   private String author;

   public FastDFSFile(String name, byte[] content, String ext, String height,
                  String width, String author) {
      super();
      this.name = name;
      this.content = content;
      this.ext = ext;
      this.author = author;
   }

   public FastDFSFile(String name, byte[] content, String ext) {
      super();
      this.name = name;
      this.content = content;
      this.ext = ext;

   }

   public String getName() {
      return name;
   }

   public void setName(String name) {
      this.name = name;
   }

   public byte[] getContent() {
      return content;
   }

   public void setContent(byte[] content) {
      this.content = content;
   }

   public String getExt() {
      return ext;
   }

   public void setExt(String ext) {
      this.ext = ext;
   }

   public String getMd5() {
      return md5;
   }

   public void setMd5(String md5) {
      this.md5 = md5;
   }

   public String getAuthor() {
      return author;
   }

   public void setAuthor(String author) {
      this.author = author;
   }
}

6，启动类：

package com.example.fastdfs;

import org.apache.coyote.http11.AbstractHttp11Protocol;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer;
import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
import org.springframework.context.annotation.Bean;

@SpringBootApplication
public class FastdfsApplication {

   public static void main(String[] args) {
      SpringApplication.run(FastdfsApplication.class, args);
   }

   @Bean
   public TomcatServletWebServerFactory tomcatEmbedded() {
      TomcatServletWebServerFactory tomcat = new TomcatServletWebServerFactory();
      tomcat.addConnectorCustomizers((TomcatConnectorCustomizer) connector -> {
         if ((connector.getProtocolHandler() instanceof AbstractHttp11Protocol)) {
            //-1 means unlimited
            ((AbstractHttp11Protocol) connector.getProtocolHandler()).setMaxSwallowSize(-1);
         }
      });
      return tomcat;
   }
}

7，启动，上传后结果：

马赛克部分为你的ip。