文件分片上传

概述：
最近有个文件上传的优化需求，由于文件比较大，在网络比较差的时候，文件上传经常超时，故改为将文件分片上传。

说明：
1.由于文件服务很简单，没有任何引入任何类型的数据库，所以不支持断点续传（断点续传需要使用数据库等方式保存已经上传的分片索引，最后再合并文件并校验）
2.若是多实例部署，需要实时的文件同步工具，或者网关转发请求时可以根据客户端IP（的Hash）进行转发，保证文件在一次上传过程中，处理该条请求的服务器不会发生变化
3.未对文件的大小做严格的限制，目前仅依据前端传入的参数进行文件尺寸的前置校验，若文件大小超出限制且前端参数被篡改，需要在达到限制后才能发现
4.要实现对资源的访问，还需要配置nginx等反向代理工具，对资源进行映射

Nginx反向代理配置（监听80端口，如果使用https，则需要监听443端口）:

        location / {
            alias   D://upload/file/;
        }

依赖：

        
            com.squareup.okhttp3
            okhttp
            3.11.0
            
                
                    com.google.android
                    android
                
            
        
        
            org.springframework.boot
            spring-boot-starter-validation
        

代码：

VelificationUtil

package com.example.study.util;

import lombok.extern.slf4j.Slf4j;

import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

@Slf4j
public class VelificationUtil {
    private static final String MD5 = "MD5";
    private static final String EMPTY = "";
    private static final byte[] HEX = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
    private static final byte LOW = 0xf;

    /**
     * 获取byte数组的md5
     *
     * @param bytes byte数组
     * @return md5
     */
    public static String getMD5Hex(byte[] bytes) {
        try {
            MessageDigest md5 = MessageDigest.getInstance(MD5);
            md5.update(bytes);
            byte[] digest = md5.digest();
            return byte2Hex(digest);
        } catch (NoSuchAlgorithmException e) {
            log.error("加密方法{}不支持：{}", MD5, e);
        }
        return EMPTY;
    }

    /**
     * byte数组转16进制字符串
     *
     * @param bytes byte数组
     * @return 16进制字符串
     */
    private static String byte2Hex(byte[] bytes) {
        int length = bytes.length;
        byte[] hexs = new byte[length << 1];
        for (int index = 0; index < length; index++) {
            hexs[index << 1] = HEX[bytes[index] >>> 4 & LOW];
            hexs[(index << 1) + 1] = HEX[bytes[index] & LOW];
        }
        return new String(hexs);
    }
}

FileSplitUploadStatusEnum

package com.example.study.enums;

import lombok.AllArgsConstructor;
import lombok.Getter;

/**
 * 文件分片上传状态码
 * http状态码见：http://tools.jb51.net/table/http_status_code
 */
@Getter
@AllArgsConstructor
public enum FileSplitUploadStatusEnum {
    SUCCESS(200, 1000, "整个文件上传完成"),
    SPLIT_SUCCESS(100, 1001, "分片上传完成，请继续上传后续分片"),
    PARAM_VALID_FAIL(400, 1002, "文件参数校验失败"),
    FILE_INFO_CONFLICT(409, 1003, "当前上传人数较多，文件冲突，请修改文件名或稍后上传"),
    OVER_MAX_SIZE(424, 1004, "文件大小超出限制"),
    FILE_MODIFIED_SIZE(424, 1005, "文件被篡改-文件长度不符"),
    FILE_MODIFIED_MD5(424, 1006, "文件被篡改-md5校验失败"),
    SYSTEM_ERROR(500, 1007, "系统异常");

    /**
     * 通用的http状态码
     */
    Integer code;

    /**
     * 上传文件分片的内部状态码
     */
    Integer status;

    /**
     * 状态描述
     */
    String msg;
}

SplitFileInfoRequest

package com.example.study.vo;

import lombok.Data;

import javax.validation.constraints.*;

/**
 * 注：这里我使用的Spring Boot的版本为2.4.0，以下对参数的注解的message属性值，均不会返回给前端，若需要返回，请使用其他版本的javax.validation
 */
@Data
public class SplitFileInfoRequest {
    /**
     * 文件名称
     */
    @NotBlank(message = "无效的文件名，请重新上传")
    private String fileName;

    /**
     * 整个文件的md5
     */
    @NotBlank(message = "无效的文件md5，请重新上传")
    private String fileMd5;

    /**
     * 文件总大小
     * 由于需要读取整个文件到byte数组中以获取md5值，所以文件大小不能超过byte数组大小的限制（即Integer.MAX_VALUE）
     */
    @NotNull(message = "文件总大小不能为空")
    @Min(value = 1, message = "无效的文件总大小，请重新上传")
    @Max(value = Integer.MAX_VALUE, message = "文件总大小超出限制")
    private Integer totalSize;

    /**
     * 总分片数
     */
    @NotNull(message = "总分片数不能为空")
    @Min(value = 1, message = "无效的总分片数，请重新上传")
    private Integer splitTotal;

    /**
     * 当前为第几分片（从1开始）
     */
    @NotNull(message = "分片索引不能为空")
    @Min(value = 1, message = "无效的分片索引，请重新上传")
    private Integer splitIndex;

    /**
     * 当前分片大小
     */
    @NotNull(message = "当前分片大小不能为空")
    @Min(value = 1, message = "文件分片大小为1B-512KB")
    @Max(value = 524288, message = "文件分片大小为1B-512KB")
    private Integer splitSize;

    /**
     * 当前文件分片内容(最大512k)
     */
    @NotNull(message = "当前文件分片内容不能为空")
    @Size(min = 1, max = 524288, message = "文件分片大小限制为1B-512KB")
    private byte[] splitData;

    public boolean isFirstSplit() {
        return splitIndex.equals(1);
    }

    public boolean isLastSplit() {
        return splitIndex.equals(splitTotal);
    }
}

FileSplitUploadResponse

package com.example.study.vo;

import com.example.study.enums.FileSplitUploadStatusEnum;
import lombok.*;

@Setter
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileSplitUploadResponse {
    private static final String SPLIT_SUCCESS_MSG = "第%d分片上传完成，共%d分片。请继续上传后续分片";
    private Integer code;
    private Integer status;
    private String msg;
    private T data;

    public FileSplitUploadResponse(FileSplitUploadStatusEnum statusEnum) {
        this.code = statusEnum.getCode();
        this.status = statusEnum.getStatus();
        this.msg = statusEnum.getMsg();
    }

    public FileSplitUploadResponse(Integer splitIndex, Integer splitTotal) {
        FileSplitUploadStatusEnum statusEnum = FileSplitUploadStatusEnum.SPLIT_SUCCESS;
        this.code = statusEnum.getCode();
        this.status = statusEnum.getStatus();
        this.msg = String.format(SPLIT_SUCCESS_MSG, splitIndex, splitTotal);
    }
}

CustomConfig

package com.example.study.config;

import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;

@Getter
@Configuration
public class CustomConfig {
    /**
     * 服务的url，用于拼接访问上传后的资源
     */
    @Value("${file.upload.server.url:http://localhost/}")
    private String fileUploadServerUrl;

    /**
     * 本地文件存放路径(Linux)
     */
    @Value("${file.upload.path.linux:/upload/file/}")
    private String fileUploadInLinux;

    /**
     * 本地文件存放路径(Windows)
     */
    @Value("${file.upload.path.windows:D:/upload/file/}")
    private String fileUploadInWindows;

    /**
     * 文件上传最大时间（毫秒）
     * 默认十分钟
     */
    @Value("${file.upload.max.time:600000}")
    private long maxUploadTime;

    /**
     * 文件上传大小限制
     * 默认100M
     */
    @Value("${file.upload.max.size:104857600}")
    private long maxFileSize;
}

SplitFileParam

package com.example.study.param;

import com.example.study.vo.SplitFileInfoRequest;
import lombok.Getter;
import lombok.Setter;

/**
 * 用于保存临时文件的参数信息
 */
@Getter
@Setter
public class SplitFileParam {
    /**
     * 临时文件的文件名
     */
    private String tmpFileName;

    /**
     * 临时文件的本地存储路径
     */
    private String tmpLocalPath;

    /**
     * 最终的文件名
     */
    private String fileName;

    /**
     * 文件本地存储路径
     */
    private String localPath;

    /**
     * 文件给外部访问的url
     */
    private String webUrl;

    /**
     * 前端传入的请求体（文件参数）
     */
    private SplitFileInfoRequest splitFileInfo;
}

 FileSplitUploadService

package com.example.study.service;

import com.example.study.vo.FileSplitUploadResponse;
import com.example.study.vo.SplitFileInfoRequest;

/**
 * 文件分片上传
 */
public interface FileSplitUploadService {
    /**
     * 文件分片上传
     *
     * @param splitFileInfo 文件信息
     * @return 上传结果
     */
    FileSplitUploadResponse splitUploadFile(SplitFileInfoRequest splitFileInfo);
}

 FileSplitUploadServiceImpl

package com.example.study.service.impl;

import com.example.study.config.CustomConfig;
import com.example.study.enums.FileSplitUploadStatusEnum;
import com.example.study.param.SplitFileParam;
import com.example.study.service.FileSplitUploadService;
import com.example.study.util.VelificationUtil;
import com.example.study.vo.FileSplitUploadResponse;
import com.example.study.vo.SplitFileInfoRequest;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributeView;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.UUID;

@Slf4j
@Service
public class FileSplitUploadServiceImpl implements FileSplitUploadService {
    private static final String PROPERTY_OS_NAME = "os.name";
    private static final String OS_LINUX = "linux";
    private static final String FILE_NAME_SEPARATOR = "-";
    private static final String TMP_FILE_NAME_SUFFIX = ".tmp";

    @Autowired
    private CustomConfig config;

    @Override
    public FileSplitUploadResponse splitUploadFile(SplitFileInfoRequest splitFileInfo) {
        FileSplitUploadResponse response = null;
        SplitFileParam param = getParam(splitFileInfo);
        try {
            // 只有第一个分片时需要校验目录、文件是否存在。若请求有问题，一开始就从非第一分片开始传，直接抛出异常即可
            if (splitFileInfo.isFirstSplit()) {
                File tmpFile = new File(param.getTmpLocalPath());
                // 建立文件目录
                if (!tmpFile.getParentFile().exists()) {
                    tmpFile.getParentFile().mkdirs();
                }
                if (tmpFile.exists() && isConfilct(param.getTmpLocalPath())) {
                    log.warn("文件冲突，文件名：{}", param.getTmpFileName());
                    response = new FileSplitUploadResponse<>(FileSplitUploadStatusEnum.FILE_INFO_CONFLICT);
                    return response;
                }
            }
            writeFile(param.getSplitFileInfo().getSplitData(), param.getTmpLocalPath());
            if (splitFileInfo.isLastSplit()) {
                int checkResult = checkAndRenameFile(param);
                switch (checkResult) {
                    case 0:
                        response = new FileSplitUploadResponse(FileSplitUploadStatusEnum.SUCCESS);
                        response.setData(param.getWebUrl());
                        break;
                    case 1:
                        response = new FileSplitUploadResponse(FileSplitUploadStatusEnum.FILE_MODIFIED_SIZE);
                        break;
                    case 2:
                        response = new FileSplitUploadResponse(FileSplitUploadStatusEnum.FILE_MODIFIED_MD5);
                        break;
                    default:
                        log.error("check result:{}", checkResult);
                }
            } else {
                log.info("保存文件{}的第{}分片成功", param.getTmpFileName(), param.getSplitFileInfo().getSplitIndex());
                response = new FileSplitUploadResponse(splitFileInfo.getSplitIndex(), splitFileInfo.getSplitTotal());
            }
        } catch (Exception e) {
            log.error("文件上传失败: ", e);
            deleteFile(new File(param.getTmpLocalPath()));
            if (e.getMessage().equals(FileSplitUploadStatusEnum.OVER_MAX_SIZE.getCode().toString())) {
                response = new FileSplitUploadResponse(FileSplitUploadStatusEnum.OVER_MAX_SIZE);
            } else {
                response = new FileSplitUploadResponse(FileSplitUploadStatusEnum.SYSTEM_ERROR);
            }
        }
        return response;
    }

    /**
     * 校验文件并重命名
     * 为了减少对象的频繁创建与销毁，把校验和重命名放在一起
     *
     * @param param 文件参数
     * @return 0-成功/1-文件大小不符/2-文件md5不符
     * @throws IOException IO异常
     */
    private int checkAndRenameFile(SplitFileParam param) throws IOException {
        File file = new File(param.getTmpLocalPath());
        // check lenth
        if (param.getSplitFileInfo().getTotalSize() - file.length() != 0) {
            log.warn("文件大小不符：{}", param.getTmpFileName());
            deleteFile(file);
            return 1;
        }
        // check md5
        byte[] fileBytes = new byte[(int) file.length()];
        InputStream is = new FileInputStream(file);
        is.read(fileBytes);
        String md5Hex = VelificationUtil.getMD5Hex(fileBytes);
        is.close();
        if (!md5Hex.equals(param.getSplitFileInfo().getFileMd5())) {
            log.warn("文件md5不符：{}", param.getTmpFileName());
            deleteFile(file);
            return 2;
        }
        log.info("文件校验成功：{}", param.getTmpFileName());
        if (file.renameTo(new File(param.getLocalPath()))) {
            log.info("文件重命名成功：{} -> ", param.getTmpFileName(), param.getFileName());
        }
        return 0;
    }

    /**
     * 写入分片的数据到文件中
     *
     * @param splitData   分片数据
     * @param tmpFilePath 临时文件路径
     * @throws IOException IO异常
     */
    private void writeFile(byte[] splitData, String tmpFilePath) throws IOException {
        RandomAccessFile randomAccessFile = new RandomAccessFile(tmpFilePath, "rw");
        if (randomAccessFile.length() + splitData.length > config.getMaxFileSize()) {
            randomAccessFile.close();
            throw new RuntimeException(FileSplitUploadStatusEnum.OVER_MAX_SIZE.getCode().toString());
        }
        randomAccessFile.seek(randomAccessFile.length());
        randomAccessFile.write(splitData);
        randomAccessFile.close();
    }

    /**
     * 获取保存文件需要的参数
     *
     * @param splitFileInfo 前端上传的文件信息
     * @return 保存文件需要的参数
     */
    private SplitFileParam getParam(SplitFileInfoRequest splitFileInfo) {
        SplitFileParam param = new SplitFileParam();
        // 文件保存路径（OS下的路径）
        String localPathPre = System.getProperty(PROPERTY_OS_NAME).equalsIgnoreCase(OS_LINUX)
                ? config.getFileUploadInLinux() : config.getFileUploadInWindows();
        // 临时文件名
        String tmpFileName = splitFileInfo.getFileMd5()
                .concat(FILE_NAME_SEPARATOR)
                .concat(splitFileInfo.getFileName())
                .concat(TMP_FILE_NAME_SUFFIX);
        param.setTmpFileName(tmpFileName);
        param.setTmpLocalPath(localPathPre.concat(tmpFileName));
        // 最终的文件名
        if (splitFileInfo.isLastSplit()) {
            String fileName = UUID.randomUUID().toString()
                    .concat(FILE_NAME_SEPARATOR)
                    .concat(splitFileInfo.getFileName());
            param.setFileName(fileName);
            param.setLocalPath(localPathPre.concat(fileName));
            String webUrl = config.getFileUploadServerUrl().concat(fileName);
            param.setWebUrl(webUrl);
        }
        param.setSplitFileInfo(splitFileInfo);
        return param;
    }

    /**
     * 文件是否冲突
     * 如果md5和文件名都相同，就会产生冲突
     * 根据文件最大上传时间来判断到底是冲突还是脏数据，如果是脏数据，删除脏数据
     *
     * @param filePath 文件路径
     * @return false-脏数据/true-文件冲突
     */
    private boolean isConfilct(String filePath) {
        Long fileCreateTimeInMills = getFileCreateTimeInMills(filePath);
        if (System.currentTimeMillis() - fileCreateTimeInMills > config.getMaxUploadTime()) {
            deleteFile(new File(filePath));
            return false;
        }
        return true;
    }

    /**
     * 删除文件
     *
     * @param file 待删除文件
     */
    private void deleteFile(File file) {
        boolean delete = file.delete();
        if (!delete) {
            log.warn("文件删除失败:{}", file.getPath());
        }
    }

    /**
     * 获取文件创建时间
     *
     * @param filePath 文件路径
     * @return 文件创建时间戳
     */
    private static Long getFileCreateTimeInMills(String filePath) {
        try {
            Path path = Paths.get(filePath);
            BasicFileAttributeView basicview = Files.getFileAttributeView(path, BasicFileAttributeView.class, LinkOption.NOFOLLOW_LINKS);
            BasicFileAttributes attr = basicview.readAttributes();
            return attr.creationTime().toMillis();
        } catch (Exception e) {
            log.warn("获取文件创建时间出错：filePath:{}, errorMsg:{}", filePath, e);
            File file = new File(filePath);
            return file.lastModified();
        }
    }
}

SpiltUploadController 

package com.example.study.controller;

import com.example.study.enums.FileSplitUploadStatusEnum;
import com.example.study.service.FileSplitUploadService;
import com.example.study.vo.FileSplitUploadResponse;
import com.example.study.vo.SplitFileInfoRequest;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import javax.validation.Valid;


/**
 * @Author: java_t_t
 * @Description: 文件分片上传
 */
@Slf4j
@RestController
@RequestMapping("/file")
public class SpiltUploadController {
    @Autowired
    FileSplitUploadService fileSplitUploadService;

    /**
     * 上传文件到本地
     *
     * @param splitFileInfo 文件
     * @return
     */
    @PostMapping("/upload")
    public FileSplitUploadResponse uploadFileToLocal(@RequestBody @Valid SplitFileInfoRequest splitFileInfo) {
        if (splitFileInfo.getFileMd5().length() != 32
                || splitFileInfo.getSplitSize() - splitFileInfo.getSplitData().length != 0
                || splitFileInfo.getSplitIndex() > splitFileInfo.getSplitTotal()) {
            recordValidFailReason(splitFileInfo);
            FileSplitUploadResponse response = new FileSplitUploadResponse(FileSplitUploadStatusEnum.PARAM_VALID_FAIL);
            return response;
        }
        return fileSplitUploadService.splitUploadFile(splitFileInfo);
    }

    private void recordValidFailReason(SplitFileInfoRequest splitFileInfo) {
        log.info("文件信息校验失败:" +
                        "\nfileMd5:{}" +
                        "\ntotalSize:{}" +
                        "\nsplitTotal:{}" +
                        "\nsplitIndex:{}" +
                        "\nsplitSize:{}" +
                        "\nsplitData.size:{}",
                splitFileInfo.getFileMd5(),
                splitFileInfo.getTotalSize(),
                splitFileInfo.getSplitTotal(),
                splitFileInfo.getSplitIndex(),
                splitFileInfo.getSplitSize(),
                splitFileInfo.getSplitData().length);
    }
}

 

 

 

 

测试类-TestUpload

package com.example.study.common;

import com.alibaba.fastjson.JSON;
import com.example.study.util.VelificationUtil;
import com.example.study.vo.SplitFileInfoRequest;
import okhttp3.*;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.concurrent.TimeUnit;

public class TestUpload {
    private static OkHttpClient client;
    private static final String url = "http://localhost:8080/file/upload";
    private static MediaType mediaType = MediaType.parse("application/json; charset=utf-8");

    public static void main(String[] args) throws IOException {
        client = new OkHttpClient.Builder()
                .connectTimeout(20, TimeUnit.SECONDS)
                .readTimeout(20, TimeUnit.SECONDS)
                .build();


        String filePath = "E:\\360安全浏览器下载\\HUAWEICLOUDMeeting_Win.exe";
        File file = new File(filePath);
        RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r");
        byte[] splitData = new byte[524288];
        long totalSize = randomAccessFile.length();
        long splitTotal = totalSize / 524288;
        if (totalSize % 524288 != 0) {
            splitTotal++;
        }
        SplitFileInfoRequest body = new SplitFileInfoRequest();
        body.setFileName(file.getName());
        byte[] wholeData = new byte[(int) file.length()];
        new FileInputStream(file).read(wholeData);
        body.setFileMd5(VelificationUtil.getMD5Hex(wholeData));
        body.setTotalSize((int) file.length());
        body.setSplitTotal((int) splitTotal);

        int index = 1;
        int readSize;
        while ((readSize = randomAccessFile.read(splitData)) != -1) {
            body.setSplitSize(readSize);
            body.setSplitIndex(index);
            body.setSplitData(splitData);

            if (readSize != 524288) {
                byte[] lastSplitData = new byte[readSize];
                System.arraycopy(splitData, 0, lastSplitData, 0, readSize);
                body.setSplitData(lastSplitData);
            }
            Request request = new Request.Builder()
                    .url(url)
                    .post(RequestBody.create(mediaType, JSON.toJSONString(body)))
                    .build();
            Response execute = client.newCall(request).execute();
            System.out.println(execute.body().string());
            index++;
        }
        System.out.println("finished");
    }
}

 

 

测试结果：

反向代理结果：