微服务笔记之Spring Cloud 中使用MongoDB GridFS实现文件存储服务(Finchley)

一、 说明

1、MongoDB GridFS简介

GridFS 用于存储和恢复那些超过16M（BSON文件限制）的文件(如：图片、音频、视频等)。

GridFS 也是文件存储的一种方式，但是它是存储在MonoDB的集合中。

GridFS 可以更好的存储大于16M的文件。

GridFS 会将大文件对象分割成多个小的chunk(文件片段),一般为256k/个,每个chunk将作为MongoDB的一个文档(document)被存储在chunks集合中。

GridFS 用两个集合来存储一个文件：fs.files与fs.chunks。

每个文件的实际内容被存在chunks(二进制数据)中,和文件有关的meta数据(filename,content_type,还有用户自定义的属性)将会被存在files集合中。

2、目的：利用spring-boot和mongodb gridfs实现文件上传及下载服务

3、配置版本:

       spring-boot:  2.0.3.RELEASE

       spring-cloud：Finchley.RELEASE

二、 项目及代码

    创建maven项目：module名称 gridfsfile

 

  

1、pom文件配置如下

  4.0.0

  com.pasq.base.service
  gridfs-file
  1.0-SNAPSHOT

  gridfs-file
  
  http://www.example.com

    
        org.springframework.boot
        spring-boot-starter-parent
        2.0.3.RELEASE
        
    


    
        UTF-8
        1.7
        1.7
        UTF-8
        1.8
        Finchley.RELEASE
    

  
    
      junit
      junit
      4.11
      test
    
      
          org.springframework.boot
          spring-boot-starter-web
          
      

      
          org.springframework.cloud
          spring-cloud-starter-netflix-eureka-client
      

      
      
          org.apache.commons
          commons-io
          1.3.2
      
      
      
          org.springframework.boot
          spring-boot-starter-data-mongodb
          
      

  

    
        
            
                org.springframework.cloud
                spring-cloud-dependencies
                ${spring-cloud.version}
                pom
                import
            
        
    

  
    
      
        
          maven-clean-plugin
          3.0.0
        
      
    
  

   2、 spring-boot的application.properties配置:   

spring.application.name=gridfs-server

##server
server.port=9909
server.tomcat.uri-encoding=UTF-8

eureka.client.serviceUrl.defaultZone=http://localhost:8761/eureka/
##http 编码配置
spring.http.encoding.charset=UTF-8
spring.http.encoding.enabled=true
spring.http.encoding.force=true

#mongodb
spring.data.mongodb.host=192.168.200.222
spring.data.mongodb.port=27017
spring.data.mongodb.database=gridfs
spring.data.mongodb.usernmae=cccc
spring.data.mongodb.password=asd123

  3.  mongo连接配置文件MongoConf.java,使用spring Configuration注解配置  

package com.pasq.base.service.gridfs.conf;

import com.mongodb.client.MongoDatabase;
import com.mongodb.client.gridfs.GridFSBucket;
import com.mongodb.client.gridfs.GridFSBuckets;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.mongodb.MongoDbFactory;

/**
 * @author lcl
 * @version 0.1
 * @date 2018/11/13
 **/
@Configuration
public class MongoConf {

    @Autowired
    private MongoDbFactory mongoDbFactory;


    @Bean
    public GridFSBucket getGridFSBucket(){
        MongoDatabase db = mongoDbFactory.getDb();
        return GridFSBuckets.create(db);
    }
}

4、  服务API: GfsFileController代码(此处没有写service层)

    此处上传下载各使用了几种不同的方式，可参见各自注解

package com.pasq.base.service.gridfs;

import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import com.mongodb.client.gridfs.GridFSBucket;
import com.mongodb.client.gridfs.GridFSDownloadStream;
import com.mongodb.client.gridfs.model.GridFSFile;
import com.pasq.base.service.pojo.FileResponse;
import com.pasq.base.service.pojo.MyResponse;
import org.apache.commons.io.IOUtils;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.Resource;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.gridfs.GridFsResource;
import org.springframework.data.mongodb.gridfs.GridFsTemplate;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.UUID;

import static org.springframework.data.mongodb.core.query.Query.query;
import static org.springframework.data.mongodb.gridfs.GridFsCriteria.whereFilename;

/**
 * 上传下载
 * @author lcl
 * @version 0.1
 * @date 2018/11/13
 **/
@RestController
@RequestMapping("/")
public class GfsFileController {

    Logger logger = LoggerFactory.getLogger(GfsFileController.class);

    @Autowired
    private GridFsTemplate gridFsTemplate;

    @Autowired
    private GridFSBucket gridFSBucket;



    /**
     * 上传文件，由程序生成新的uuid名,后缀同原文件后缀
     * @param file MultipartFile
     * @return FileResponse 只包含生成的唯一文件名
     */
    @RequestMapping(value = "upload",method = RequestMethod.POST,produces = MediaType.APPLICATION_JSON_VALUE)
    public FileResponse saveFile(@RequestParam(value = "file",required = true) MultipartFile file){
        String fullName = file.getOriginalFilename();
        logger.debug("Save File..."+fullName);
        String endFile = fullName.substring(fullName.lastIndexOf("."));
        DBObject dbObject = new BasicDBObject();
        ((BasicDBObject) dbObject).put("createdDate",new Date());

        String fileName = UUID.randomUUID().toString().replace("-","")+endFile;
        logger.debug("File Name:"+fileName);
        InputStream inputStream = null;
        try {
          inputStream = file.getInputStream();
          gridFsTemplate.store(inputStream,fileName,dbObject);
          logger.debug("Saved File:"+fileName);
        }catch (IOException e){
          logger.error(e.getMessage());
        }
        FileResponse fileResponse = new FileResponse();
        fileResponse.setFileName(fileName);
        fileResponse.setOriginalFilename(fullName);
        return fileResponse;
    }

    /**
     * 上传文件,文件唯一标识使用mongodb的ObjectId,存入时保留原始文件名
     * @param file MultipartFile
     * @return FileResponse：文件唯一ID,原始文件名等
     */
    @RequestMapping(value = "fileId/upload",method = RequestMethod.POST,produces = MediaType.APPLICATION_JSON_VALUE)
    @ResponseBody
    public MyResponse saveFileV2(@RequestParam(value = "file",required = true) MultipartFile file){
        String fullName = file.getOriginalFilename();
        logger.debug("Save File..."+fullName);
        FileResponse fileResponse = new FileResponse();
        ObjectId fileId = null;
        DBObject dbObject = new BasicDBObject();
        ((BasicDBObject) dbObject).put("createdDate",new Date());

        MyResponse myResponse = new MyResponse();

        logger.debug("File Name:"+fullName);
        InputStream inputStream = null;
        try {
            inputStream = file.getInputStream();
            fileId = gridFsTemplate.store(inputStream,fullName,dbObject);
            logger.debug("Saved File:"+fullName);
        }catch (IOException e){
            logger.error(e.getMessage());
            myResponse.setCode(100);
            myResponse.setMsg("保存文件失败");
            return myResponse;
        }
        fileResponse.setFileName(fullName);
        fileResponse.setOriginalFilename(fullName);
        fileResponse.setContentType(file.getContentType());
        fileResponse.setFileId(fileId==null?null:fileId.toString());

        myResponse.setCode(0);
        myResponse.setMsg("success");
        myResponse.setData(fileResponse.getFileId());
        return myResponse;
    }

    /**
     * @deprecated  弃用
     * @param fileName
     * @return 直接返回字节流
     * @throws IOException
     */
    @RequestMapping(value = "test/download",method = RequestMethod.GET)
    public byte[] getFileTest(@RequestParam(value = "file",required = true) String fileName) throws IOException{
        logger.debug("Get File..."+fileName);
        GridFSFile file = gridFsTemplate.findOne(query(whereFilename().is(fileName)));

        if(file==null){
          logger.error("No File.");
          throw new RuntimeException("No file:"+fileName);
        }
        GridFSDownloadStream in = gridFSBucket.openDownloadStream(file.getObjectId());
        GridFsResource resource = new GridFsResource(file,in);
        InputStream inputStream = resource.getInputStream();
        return IOUtils.toByteArray(inputStream);
    }


    /**
     *使用由程序生成的唯一文件名下载获取文件
     * @param fileName 唯一文件名 uuid.后缀
     * @param request
     * @return
     * @throws IOException
     */
    @RequestMapping(value = "download",method = RequestMethod.GET)
    public ResponseEntity getFile(@RequestParam(value = "file",required = true) String fileName,HttpServletRequest request) throws IOException{
        logger.debug("Get File..."+fileName);
        GridFSFile file = gridFsTemplate.findOne(query(whereFilename().is(fileName)));

        if(file==null){
            logger.error("No File.");
            throw new RuntimeException("No file:"+fileName);
        }
        GridFSDownloadStream in = gridFSBucket.openDownloadStream(file.getObjectId());
        GridFsResource resource = new GridFsResource(file,in);
        String contentType = null;
        try {
            contentType = request.getServletContext().getMimeType(resource.getFile().getAbsolutePath());
        } catch (IOException ex) {
            logger.info("Could not determine file type.");
        }
        if(contentType == null) {
            contentType = "application/octet-stream";
        }

        return ResponseEntity.ok().contentType(MediaType.parseMediaType(contentType))
                .header(HttpHeaders.CONTENT_DISPOSITION,"attachment; filename=\""+fileName+"\"")
                .body((Resource) resource);
    }

    /**
     * 使用mongodb的ObjectId（的id）下载文件，此时文件以上传时的原始名保存下载
     * @param fileId  文件在mongodb的ObjectId（的id）
     * @param request
     * @param response
     * @throws IOException
     */
    @RequestMapping(value = "fileId/download",method = RequestMethod.GET)
    public void getFileById(@RequestParam(value = "fileId",required = true) String fileId, HttpServletRequest request, HttpServletResponse response) throws IOException{
        logger.debug("Download Request File Id:"+fileId);
        Query query = Query.query(Criteria.where("_id").is(fileId));
        GridFSFile file = gridFsTemplate.findOne(query);
        if(file==null){
            logger.error("No File Id:"+fileId);
            throw new RuntimeException("No file Id:"+fileId);
        }
        GridFSDownloadStream in = gridFSBucket.openDownloadStream(file.getObjectId());
        GridFsResource resource = new GridFsResource(file,in);
        String fileName = file.getFilename().replace(",", "");
        //处理中文文件名乱码
        String userAgent = request.getHeader("User-Agent").toUpperCase();
        logger.debug("Download Request User-Agent:"+userAgent);

        if (userAgent.contains("MSIE") ||
                userAgent.contains("TRIDENT")
                || userAgent.contains("EDGE")
                || userAgent.contains("CHROME")
                || userAgent.contains("SAFARI")
                || userAgent.contains("MOZILLA")) {
            fileName = java.net.URLEncoder.encode(fileName, "UTF-8");
        } else {
            //非IE浏览器的处理：
//            fileName = new String(fileName.getBytes("UTF-8"), "ISO-8859-1");
            fileName = java.net.URLEncoder.encode(fileName, "UTF-8");
        }
        // 通知浏览器进行文件下载
        response.setHeader("Content-Disposition", "attachment;filename=\"" + fileName + "\"");
        IOUtils.copy(resource.getInputStream(),response.getOutputStream());
    }

    /**
     * 根据文件ID(mongodb中的唯一ID)删除文件
     * @param fileId
     * @return
     */
    @RequestMapping(value = "fileId/delete",method = RequestMethod.POST)
    public MyResponse deleteFile(@RequestParam(value = "fileId") String fileId){

        Query query = Query.query(Criteria.where("_id").is(fileId));
        // 查询单个文件
        GridFSFile gfsfile = gridFsTemplate.findOne(query);

        MyResponse myResponse = new MyResponse();
        if (gfsfile == null) {
            myResponse.setCode(300);
            myResponse.setMsg("未找到要删除的文件。");
            return myResponse;
        }
        gridFsTemplate.delete(query);

        myResponse.setCode(0);
        myResponse.setMsg("success");
        myResponse.setData(gfsfile.getObjectId().toString());
        return myResponse;
    }


    /**
     * 使用mongodb的ObjectId（的id）下载文件，此时文件以上传时的原始名保存下载,
     *此下载服务可配和spring-cloud中使用feign-client客户端来使用，使用方式参照其他文章
     * @param fileId  文件在mongodb的ObjectId（的id）
     * @param request
     * @throws IOException
     * @return ResponseEntity
     */
    @RequestMapping(value = "oss",method = RequestMethod.GET)
    public ResponseEntity getFileByIdOss(@RequestParam(value = "fileName") String fileId, HttpServletRequest request) throws IOException{
        logger.debug("Download Request File Id:"+fileId);
        Query query = Query.query(Criteria.where("_id").is(fileId));
        GridFSFile file = gridFsTemplate.findOne(query);
        if(file==null){
            logger.error("No File Id:"+fileId);
            throw new RuntimeException("No file Id:"+fileId);
        }

        ResponseEntity entity = null;
        HttpHeaders headers = new HttpHeaders();

        GridFSDownloadStream in = gridFSBucket.openDownloadStream(file.getObjectId());
        GridFsResource resource = new GridFsResource(file,in);
        String fileName = file.getFilename().replace(",", "");
        //处理中文文件名乱码
        String userAgent = request.getHeader("User-Agent").toUpperCase();
        logger.debug("Download Request User-Agent:"+userAgent);

        if (userAgent.contains("MSIE") ||
                userAgent.contains("TRIDENT")
                || userAgent.contains("EDGE")
                || userAgent.contains("CHROME")
                || userAgent.contains("SAFARI")
                || userAgent.contains("MOZILLA")) {
            fileName = java.net.URLEncoder.encode(fileName, "UTF-8");
        } else {
            //非IE浏览器的处理：
//            fileName = new String(fileName.getBytes("UTF-8"), "ISO-8859-1");
            fileName = java.net.URLEncoder.encode(fileName, "UTF-8");
        }

        headers.add("Content-Disposition", "attachment;filename="+fileName);
        HttpStatus status = HttpStatus.OK;
        // 封装响应
        InputStream inputStream = resource.getInputStream();
        entity = new ResponseEntity<>(IOUtils.toByteArray(inputStream), headers, status);
        return entity;
    }
}

 FileResponse和MyResponse只是为响应定义的bean,可根据实际自行定义；

5、  Sping Boot 主启动类GridfsServerApplication.java 

package com.pasq.base.service;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.netflix.eureka.EnableEurekaClient;

/**
 * Hello world!
 *
 */

@SpringBootApplication
@EnableEurekaClient
public class GridfsServerApplication
{
    public static void main( String[] args )
    {
        SpringApplication.run(GridfsServerApplication.class,args);
    }
}

6、可通过postman或浏览器验证上传下载的api.