分片文件异步合并上传
对于大文件，为了避免上传导致网络带宽不够用，还有避免内存溢出，我们采用分片异步上传。
controller层，在前端对文件进行分片，先计算文件md5码，方便后续文件秒传，然后再计算可以分成多少个分片，得到分片总数以及当前分片下标。
@RequestMapping("/uploadFile")
@SentinelResource(
value = "uploadFile",
blockHandler = "uploadBlock",
blockHandlerClass = {SentinelBlockHandler.class} // 限流处理类
)
public ResponseVO uploadFile(HttpSession session,
String fileId,
MultipartFile file,
@RequestParam(required = true) String fileName,
@RequestParam(required = true) String filePid,
@RequestParam(required = true) String fileMd5,
@RequestParam(required = true) Integer chunkIndex,
@RequestParam(required = true) Integer chunks) {
SessionWebUserDto webUserDto = getUserInfoFromSession(session);
UploadResultDto resultDto = fileInfoService.uploadFile(webUserDto, fileId, file, fileName, filePid, fileMd5, chunkIndex, chunks);
return getSuccessResponseVO(resultDto);
}service层
1.如果是第一个分片文件：通过md5码和当前用户id查询数据库有没有一样的，有直接复制一份
写入数据库，直接返回结果。
UploadResultDto resultDto = new UploadResultDto();
if (StringTools.isEmpty(fileId)) {
fileId = StringTools.getRandomString(Constants.LENGTH_10);
}
resultDto.setFileId(fileId);
Date curDate = new Date();
UserSpaceDto spaceDto = redisComponent.getUserSpaceUse(webUserDto.getUserId());
if (chunkIndex == 0) {
FileInfoQuery infoQuery = new FileInfoQuery();
infoQuery.setFileMd5(fileMd5);
infoQuery.setSimplePage(new SimplePage(0, 1));
infoQuery.setStatus(FileStatusEnums.USING.getStatus());
List dbFileList = this.fileInfoMapper.selectList(infoQuery);
//秒传
if (!dbFileList.isEmpty()) {
FileInfo dbFile = dbFileList.get(0);
//判断文件状态
if (dbFile.getFileSize() + spaceDto.getUseSpace() > spaceDto.getTotalSpace()) {
throw new BusinessException(ResponseCodeEnum.CODE_904);
}
dbFile.setFileId(fileId);
dbFile.setFilePid(filePid);
dbFile.setUserId(webUserDto.getUserId());
dbFile.setFileMd5(null);
dbFile.setCreateTime(curDate);
dbFile.setLastUpdateTime(curDate);
dbFile.setStatus(FileStatusEnums.USING.getStatus());
dbFile.setDelFlag(FileDelFlagEnums.USING.getFlag());
dbFile.setFileMd5(fileMd5);
fileName = autoRename(filePid, webUserDto.getUserId(), fileName);
dbFile.setFileName(fileName);
this.fileInfoMapper.insert(dbFile);
resultDto.setStatus(UploadStatusEnums.UPLOAD_SECONDS.getCode());
//更新用户空间使用
updateUserSpace(webUserDto, dbFile.getFileSize());
return resultDto;
}
} 2.如果不是最后一个分片文件，将分片文件转存到暂时目录，直接返回结果
以下是文件路径组成部分
| 临时分片 | temp/{user_id}{file_id}/ | temp/u123f456/0 | 分片按索引存储 |
| 最终文件 | files/{yyyyMM}/{user_id}{file_id}.{suffix} | files/202503/u123f456.pdf | 按时间和用户隔离存储 |
//暂存在临时目录
String tempFolderName = appConfig.getProjectFolder() + Constants.FILE_FOLDER_TEMP;
String currentUserFolderName = webUserDto.getUserId() + fileId;
//创建临时目录
tempFileFolder = new File(tempFolderName + currentUserFolderName);
if (!tempFileFolder.exists()) {
tempFileFolder.mkdirs();
}
//判断磁盘空间
Long currentTempSize = redisComponent.getFileTempSize(webUserDto.getUserId(), fileId);
if (file.getSize() + currentTempSize + spaceDto.getUseSpace() > spaceDto.getTotalSpace()) {
throw new BusinessException(ResponseCodeEnum.CODE_904);
}
File newFile = new File(tempFileFolder.getPath() + "/" + chunkIndex);
file.transferTo(newFile);
//保存临时大小
redisComponent.saveFileTempSize(webUserDto.getUserId(), fileId, file.getSize());
//不是最后一个分片，直接返回
if (chunkIndex < chunks - 1) {
resultDto.setStatus(UploadStatusEnums.UPLOADING.getCode());
return resultDto;
}3.是最后一个分片文件，注册事务同步管理器，实现在事务成功提交后调起线程的异步调用
实现分片文件转存
//最后一个分片上传完成，记录数据库，异步合并分片
String month= DateUtil.format(new Date(),DateTimePatternEnum.YYYYMM.getPattern());
String fileSuffix=StringTools.getFileSuffix(fileName);
//真实文件名
String realFileName=currentUserFolderName+fileSuffix;
FileTypeEnums fileTypeEnums=FileTypeEnums.getFileTypeBySuffix(fileSuffix);
//自动重命名
fileName=autoRename(filePid,webUserDto.getUserId(),fileName);
FileInfo fileInfo = new FileInfo();
fileInfo.setFileId(fileId);
fileInfo.setUserId(webUserDto.getUserId());
fileInfo.setFileMd5(fileMd5);
fileInfo.setFileName(fileName);
fileInfo.setFilePath(month + "/" + realFileName);
fileInfo.setFilePid(filePid);
fileInfo.setCreateTime(curDate);
fileInfo.setLastUpdateTime(curDate);
fileInfo.setFileCategory(fileTypeEnums.getCategory().getCategory());
fileInfo.setFileType(fileTypeEnums.getType());
fileInfo.setStatus(FileStatusEnums.TRANSFER.getStatus());
fileInfo.setFolderType(FileFolderTypeEnums.FILE.getType());
fileInfo.setDelFlag(FileDelFlagEnums.USING.getFlag());
fileInfoMapper.insert(fileInfo);
Long totalSize = redisComponent.getFileTempSize(webUserDto.getUserId(), fileId);
updateUserSpace(webUserDto,totalSize);
resultDto.setStatus(UploadStatusEnums.UPLOAD_FINISH.getCode());
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
fileInfoService.transferFile(fileInfo.getFileId(),webUserDto);
}
});
return resultDto;4.异步任务实现转码，需要在启动类上添加@EnableAsync
@Async
public void transferFile(String fileId, SessionWebUserDto webUserDto) {
Boolean transferSuccess = true;
String targetFilePath = null;
String cover = null;
FileTypeEnums fileTypeEnum = null;
FileInfo fileInfo = fileInfoMapper.selectByFileIdAndUserId(fileId, webUserDto.getUserId());
try {
if (fileInfo == null || !FileStatusEnums.TRANSFER.getStatus().equals(fileInfo.getStatus())) {
return;
}
//临时目录
String tempFolderName = appConfig.getProjectFolder() + Constants.FILE_FOLDER_TEMP;
String currentUserFolderName = webUserDto.getUserId() + fileId;
File fileFolder = new File(tempFolderName + currentUserFolderName);
if (!fileFolder.exists()) {
fileFolder.mkdirs();
}
//文件后缀
String fileSuffix = StringTools.getFileSuffix(fileInfo.getFileName());
String month = DateUtil.format(fileInfo.getCreateTime(), DateTimePatternEnum.YYYYMM.getPattern());
//目标目录
String targetFolderName = appConfig.getProjectFolder() + Constants.FILE_FOLDER_FILE;
File targetFolder = new File(targetFolderName + "/" + month);
if (!targetFolder.exists()) {
targetFolder.mkdirs();
}
//真实文件名
String realFileName = currentUserFolderName + fileSuffix;
//真实文件路径
targetFilePath = targetFolder.getPath() + "/" + realFileName;
//合并文件
union(fileFolder.getPath(), targetFilePath, fileInfo.getFileName(), true);
//视频文件切割
fileTypeEnum = FileTypeEnums.getFileTypeBySuffix(fileSuffix);
if (FileTypeEnums.VIDEO == fileTypeEnum) {
cutFile4Video(fileId, targetFilePath);
//视频生成缩略图
cover = month + "/" + currentUserFolderName + Constants.IMAGE_PNG_SUFFIX;
String coverPath = targetFolderName + "/" + cover;
ScaleFilter.createCover4Video(new File(targetFilePath), Constants.LENGTH_150, new File(coverPath));
} else if (FileTypeEnums.IMAGE == fileTypeEnum) {
//生成缩略图
cover = month + "/" + realFileName.replace(".", "_.");
String coverPath = targetFolderName + "/" + cover;
Boolean created = ScaleFilter.createThumbnailWidthFFmpeg(new File(targetFilePath), Constants.LENGTH_150, new File(coverPath), false);
if (!created) {
FileUtils.copyFile(new File(targetFilePath), new File(coverPath));
}
}
} catch (Exception e) {
logger.error("文件转码失败，文件Id:{},userId:{}", fileId, webUserDto.getUserId(), e);
transferSuccess = false;
} finally {
FileInfo updateInfo = new FileInfo();
updateInfo.setFileSize(new File(targetFilePath).length());
updateInfo.setFileCover(cover);
updateInfo.setStatus(transferSuccess ? FileStatusEnums.USING.getStatus() : FileStatusEnums.TRANSFER_FAIL.getStatus());
fileInfoMapper.updateFileStatusWithOldStatus(fileId, webUserDto.getUserId(), updateInfo, FileStatusEnums.TRANSFER.getStatus());
}
}5.合并分片文件，用RandomAccessFile实现
public void union(String dirPath, String toFilePath, String fileName, boolean delSource) throws BusinessException {
File dir = new File(dirPath);
if (!dir.exists()) {
throw new BusinessException("目录不存在");
}
File fileList[] = dir.listFiles();
File targetFile = new File(toFilePath);
RandomAccessFile writeFile = null;
try {
writeFile = new RandomAccessFile(targetFile, "rw");
byte[] b = new byte[1024 * 10];
for (int i = 0; i < fileList.length; i++) {
int len = -1;
//创建读块文件的对象
File chunkFile = new File(dirPath + File.separator + i);
RandomAccessFile readFile = null;
try {
readFile = new RandomAccessFile(chunkFile, "r");
while ((len = readFile.read(b)) != -1) {
writeFile.write(b, 0, len);
}
} catch (Exception e) {
logger.error("合并分片失败", e);
throw new BusinessException("合并文件失败");
} finally {
readFile.close();
}
}
} catch (Exception e) {
logger.error("合并文件:{}失败", fileName, e);
throw new BusinessException("合并文件" + fileName + "出错了");
} finally {
try {
if (null != writeFile) {
writeFile.close();
}
} catch (IOException e) {
logger.error("关闭流失败", e);
}
if (delSource) {
if (dir.exists()) {
try {
FileUtils.deleteDirectory(dir);
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}6.trycatch包围，异常处理，直接删掉临时目录
catch (BusinessException e) {
uploadSuccess = false;
logger.error("文件上传失败", e);
throw e;
} catch (Exception e) {
uploadSuccess = false;
logger.error("文件上传失败", e);
throw new BusinessException("文件上传失败");
} finally {
//如果上传失败，清除临时目录
if (tempFileFolder != null && !uploadSuccess) {
try {
FileUtils.deleteDirectory(tempFileFolder);
} catch (IOException e) {
logger.error("删除临时目录失败");
}
}
}