spring boot - minio 分片上传
spring boot - minio 分片上传
主要分三个接口:
- 分片上传
- 分片合并
- 分片完整性校验
Md5UploadController
@RestController
public class Md5UploadController {
@Autowired
private UploadMd5Service uploadMd5Service;
/**
* 分片上传
*
* @param file 分片file
* @param fileMd5 完整fileMd5
* @param chunkIndex 分片id
* @return
*/
@PostMapping("/file/upload/chunk")
public String uploadChunk(@RequestParam("file") MultipartFile file,
@RequestParam(value = "fileMd5") String fileMd5,
@RequestParam("chunkIndex") Integer chunkIndex) {
return uploadMd5Service.uploadChunk(file, fileMd5, chunkIndex);
}
/**
* 文件合并
*
* @param fileMd5 完整 fileMd5
* @param fileName 合并后文件名称
* @param total 合并文件总数【用于校验】
* @param prefix 分片所在前缀 2023-12/09/fileMd5/chunk/chunk0...
* @return
*/
@PostMapping("/file/merge")
public String merge(@RequestParam(value = "fileMd5") String fileMd5, @RequestParam("fileName") String fileName,
@RequestParam("total") int total,
@RequestParam(value = "prefix",required = false)String prefix) {
return uploadMd5Service.merge(fileMd5, fileName, total, prefix);
}
/**
* 分片文件校验
*
* @param fileMd5 完整fileMd5
* @param chunkIndex 分片index
* @return
*/
@PostMapping("/file/upload/chunk/check")
public boolean chunkCheck(@RequestParam(value = "fileMd5") String fileMd5,
@RequestParam("chunkIndex") Integer chunkIndex) {
return uploadMd5Service.chunkCheck(fileMd5, chunkIndex);
}
}
UploadMd5Service
public interface UploadMd5Service {
/**
* 分片上传
*
* @param file 分片file
* @param fileMd5 完整fileMd5
* @param chunkIndex 分片id
* @return
*/
String uploadChunk(MultipartFile file, String fileMd5, Integer chunkIndex);
/**
* 文件合并
*
* @param fileMd5 完整 fileMd5
* @param fileName 合并后文件名称
* @param total 合并文件总数【用于校验】
* @param prefix
* @return
*/
String merge(String fileMd5, String fileName, int total, String prefix);
/**
* 分片文件校验
*
* @param fileMd5 完整fileMd5
* @param chunkIndex 分片index
* @return
*/
boolean chunkCheck(String fileMd5, Integer chunkIndex);
}
//实现
@Service
public class UploadMd5ServiceImpl implements UploadMd5Service {
@Autowired
private MinIoAuthClient minIoAuthClient;
@Autowired
private ThreadPoolTaskExecutor executor;
private static String CHUNK = "/chunk/";
private static String SUFFIX = "chunk-";
@Override
public String uploadChunk(MultipartFile file, String fileMd5, Integer chunkIndex) {
//2023-06/12/fileMd5/chunk/chunk-0,chunk-1
String objectName = getPath(fileMd5, null).concat(CHUNK).concat(SUFFIX).concat(chunkIndex.toString());
return minIoAuthClient.upload(file,objectName,true);
}
@Override
public String merge(String fileMd5, String fileName, int total, String prefix) {
//查询该 路径下所有的分片
List<String> fuzzyObject = minIoAuthClient.fuzzyListObjects(SUFFIX,getPath(fileMd5,prefix).concat(CHUNK));
if (CollectionUtils.isEmpty(fuzzyObject) || total != fuzzyObject.size()) {
throw new RuntimeException("合并失败，分片数不一致!!");
}
String pathPrefix = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM" + "/dd"));
String result = minIoAuthClient.composeObject(fuzzyObject, fileName, pathPrefix);
if (StrUtil.isNotBlank(result)){
//如果merge成功,删除分片
minIoAuthClient.deleteObject(fuzzyObject);
return pathPrefix.concat(fileMd5);
}
return null;
}
@Override
public boolean chunkCheck(String fileMd5, Integer chunkIndex) {
//2023-06/12/fileMd5/chunk/chunk-0,chunk-1
return minIoAuthClient.checkObjectExist(getPath(fileMd5, null).concat(CHUNK).concat(SUFFIX).concat(chunkIndex.toString()));
}
private String getPath(String fileMd5, String prefix){
if (StrUtil.isNotBlank(prefix)){
if (prefix.endsWith("/")){
return prefix.concat(fileMd5);
}
return prefix.concat("/").concat(fileMd5);
}
return LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM" + "/dd")).concat("/").concat(fileMd5);
}
}
MinioAuthClient
@Slf4j
@Component
public class MinIoAuthClient {
@Autowired
private MinioConfig config;
@Autowired
private MinioClient minioClient;
@Autowired
private MinioAsyncClient minioAsyncClient;
/**
* 检验文件是否存在
* @param objectName
* @return
*/
public boolean checkObjectExist(String objectName){
try (InputStream stream = minioClient.getObject(GetObjectArgs.builder()
.bucket(config.getBucketName()).object(objectName).build())) {
if (stream!=null){
return true;
}
} catch (Exception e) {
log.error("####### checkObjectExist error: {}",objectName, e);
}
return false;
}
/**
* 单个文件上传
* @param file
* @param rename
* @return
*/
public String upload(MultipartFile file,String inputFileName, Boolean rename) {
String objectName = null;
InputStream inputStream = null;
try {
String originalFilename = inputFileName!=null? inputFileName: file.getOriginalFilename();
if (!rename) {
String fileName = UUID.randomUUID().toString().replace("-", "")
+ originalFilename.substring(originalFilename.lastIndexOf("."));
objectName = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM/dd")).concat("/").concat(fileName);
} else {
objectName = originalFilename;
}
inputStream = file.getInputStream();
PutObjectArgs objectArgs = PutObjectArgs.builder().bucket(config.getBucketName()).object(objectName).stream(inputStream, file.getSize(), -1).contentType(file.getContentType()).build();
minioClient.putObject(objectArgs);
} catch (Exception e) {
e.printStackTrace();
return null;
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
log.error("delete file error", e);
}
}
}
return objectName;
}
/**
* 同步合并文件
* @param list
* @param originalFilename 合并后文件名称
* @param pathPrefix 文件所在路径
* @return
*/
public String composeObject(List<String> list, String originalFilename,String pathPrefix) {
String objectName = null;
List<ComposeSource> composeSourceList = list.stream().map(s -> {
ComposeSource source = ComposeSource.builder().bucket(config.getBucketName()).object(s).build();
return source;
}).collect(Collectors.toList());
try {
//需要按名称排序合并 xx.part0,xx.part1...
Collections.sort(composeSourceList, new Comparator<>() {
@Override
public int compare(ComposeSource o1, ComposeSource o2) {
if (o1.object().compareTo(o2.object()) < 0) {
return -1;
}
return 1;
}
});
String fileName = originalFilename;
objectName = pathPrefix.concat("/").concat(fileName);
minioClient.composeObject(ComposeObjectArgs.builder().bucket(config.getBucketName()).object(objectName).sources(composeSourceList).build());
} catch (Exception e) {
log.error("####### composeObject error", e);
return null;
}
return objectName;
}
public void deleteObject(List<String> objects){
List<DeleteObject> collect = objects.stream().map(str -> new DeleteObject(str)).collect(Collectors.toList());
Iterable<Result<DeleteError>> results = minioClient.removeObjects(RemoveObjectsArgs.builder().bucket(config.getBucketName()).objects(collect).build());
Iterator<Result<DeleteError>> iterator = results.iterator();
while (iterator.hasNext()){
Result<DeleteError> next = iterator.next();
System.out.println(next);
}
}
/**
* minio已经存在
* 获取一个指定了 HTTP 方法、到期时间和自定义请求参数的对象URL地址
*
* @param objectName
* @param timeOut
* @param timeUnit
* @return
*/
public String getPresignedObjectUrl(String objectName, int timeOut, TimeUnit timeUnit) {
try {
GetPresignedObjectUrlArgs.Builder builder = GetPresignedObjectUrlArgs.builder().method(Method.GET).bucket(config.getBucketName()).object(objectName);
if (timeOut > 0) {
builder.expiry(timeOut, timeUnit);
}
return minioClient.getPresignedObjectUrl(builder.build());
} catch (Exception e) {
log.error("###### getPresignedObjectUrl error: {}", objectName, e);
return null;
}
}
/**
* 获取文件预生成地址，【minio上文件不存在】
* 一般给合并文件使用
* @param objectName
* @param queryParams
* @param timeOut
* @param timeUnit
* @return
*/
public String getPreviewObjectUrl(String objectName,Map<String,String>queryParams, int timeOut, TimeUnit timeUnit) {
try {
GetPresignedObjectUrlArgs.Builder builder =
GetPresignedObjectUrlArgs.builder().method(Method.PUT).bucket(config.getBucketName()).object(objectName).extraQueryParams(queryParams);
if (timeOut > 0) {
builder.expiry(timeOut, timeUnit);
}
return minioClient.getPresignedObjectUrl(builder.build());
} catch (Exception e) {
log.error("###### getPreviewObjectUrl error: {}", objectName, e);
return null;
}
}
/**
* 递归根据文件名称前缀模糊查询
* @param objectName
* @return
*/
public List<String> fuzzyListObjects(String objectName) {
try {
Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(config.getBucketName()).recursive(true).build());
if (results == null || !results.iterator().hasNext()) {
return null;
}
List<String> list = new ArrayList<>();
for (Result<Item> result : results) {
String s = result.get().objectName();
String finalName = null;
if (s.lastIndexOf("/") != -1) {
finalName = s.substring(s.lastIndexOf("/") + 1);
} else {
finalName = s;
}
if (!finalName.startsWith(objectName)) {
continue;
}
list.add(s);
}
return list;
} catch (Exception e) {
log.error("#### fuzzyListObjects error: {}", objectName, e);
return null;
}
}
/**
* 模糊查询某个目录下 objectName开头的文件, 一般查询所有分片
* @param objectName 文件名称
* @param pathPrefix 路径前缀
* @return
*/
public List<String> fuzzyListObjects(String objectName,String pathPrefix) {
try {
Iterable<Result<Item>> results =
minioClient.listObjects(ListObjectsArgs.builder().bucket(config.getBucketName()).prefix(pathPrefix).build());
if (results == null || !results.iterator().hasNext()) {
return null;
}
List<String> list = new ArrayList<>();
for (Result<Item> result : results) {
String s = result.get().objectName();
String finalName = null;
if (s.lastIndexOf("/") != -1) {
finalName = s.substring(s.lastIndexOf("/") + 1);
} else {
finalName = s;
}
if (!finalName.startsWith(objectName)) {
continue;
}
list.add(s);
}
return list;
} catch (Exception e) {
log.error("#### fuzzyListObjects error: {}", objectName, e);
return null;
}
}
}
测试
上传分片
merge:
合并成功！
good luck!!