Amazon S3 Compatibility 兼容API 封装AWS S3工具类 生成预前面url跨域问题解决
Amazon S3 Compatibility 官方文档: https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm
介绍
使用Amazon S3 兼容性 API，可以继续使用他们现有的 Amazon S3 工具（例如，SDK 客户端）并对他们的应用程序进行最小的更改以使用对象存储。Amazon S3 兼容性 API和对象存储数据集是一致的。如果使用Amazon S3 Compatibility API将数据写入对象存储，则可以使用本机对象存储API 读回数据，反之亦然。
注意事项
生成URL前端跨域问题
1. Amazon(亚马逊) 解决方案:
官方解决跨域文档地址: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/userguide/enabling-cors-examples.html
2. OCI(Oracle) 解决方案: 使用OCI原生服务: API Amazon Simple Storage
官方文档地址: https://docs.amazonaws.cn/AmazonS3/latest/userguide/upload-objects.html
问题解决及工具类: https://blog.csdn.net/ayunnuo/article/details/127212315
POM依赖
<dependency>
<groupId>com.amazonawsgroupId>
<artifactId>aws-java-sdk-s3artifactId>
<version>1.12.198version>
dependency>
<dependency>
<groupId>org.projectlombokgroupId>
<artifactId>lombokartifactId>
<optional>trueoptional>
dependency>
<dependency>
<groupId>cn.hutoolgroupId>
<artifactId>hutool-allartifactId>
<version>5.8.3version>
dependency>
DTO
1. S3BaseConfig
import cn.hutool.json.JSONUtil;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* s3基础配置Bean
*
* @author yunnuo E-Mail: [email protected]
* @since 1.0.0
*/
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@ApiModel(value = "S3基础配置 Bean")
public class S3BaseConfig {
/**
* s3秘密访问密钥
*/
@ApiModelProperty(value = "s3秘密访问密钥")
private String s3SecretAccessKey;
/**
* s3访问密钥id
*/
@ApiModelProperty(value = "s3访问密钥id")
private String s3AccessKeyId;
/**
* s3 bucket
*/
@ApiModelProperty(value = "s3 bucket")
private String s3Bucket;
/**
* 地区
*/
@ApiModelProperty(value = "地区")
private String regions;
/**
* 类型 0=aws环境 1=oracle环境, default= 0
*/
@ApiModelProperty(value = "类型 0=aws环境 1=oracle环境, default=0")
private Integer type = 0;
/**
* aws环境可空， oci环境使用
*/
@ApiModelProperty(value = "namespace")
private String namespace;
@Override
public String toString() {
return JSONUtil.toJsonStr(this);
}
}
2. S3CommonConfig
import cn.hutool.json.JSONUtil;
import io.swagger.annotations.ApiModelProperty;
import lombok.*;
import lombok.experimental.Accessors;
/**
* s3 配置中心通用配置
*
* @author yunnuo Email: [email protected]
* @see S3CommonConfig : {@code 参考配置中心key: bt-user.s3.images.config}
* @since 1.0.0
*/
@Getter
@Setter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Accessors(chain = true)
public class S3CommonConfig extends S3BaseConfig {
/**
* 标题
*/
@ApiModelProperty(value = "标题")
private String title;
/**
* 描述
*/
@ApiModelProperty(value = "描述")
private String desc;
/**
* cdn前缀
*/
@ApiModelProperty(value = "cdn前缀")
private String cdnPrefix;
@Override
public String toString() {
return JSONUtil.toJsonStr(this);
}
}
AwsS3Utils 功能类
AwsS3Utils功能类, 使用的是兼容版本的api, 目前已兼容AWS (Amazon 亚马逊) 和 OCI( Oracle )两个厂商的连接配置和使用主意事项
注意: OCI 环境的桶 生产的预签名URL不支持前端跨越, 如果需要解决OCI跨域问题, 可以用OCI 原生 API Amazon Simple Storage Service 参考官方文档: https://docs.amazonaws.cn/AmazonS3/latest/userguide/upload-objects.html
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import com.amazonaws.HttpMethod;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.services.s3.transfer.model.UploadResult;
import com.kabak.rongsheng.constants.Constants;
import com.kabak.rongsheng.domain.dto.s3.S3BaseConfig;
import com.kabak.rongsheng.domain.dto.s3.S3CommonConfig;
import com.oracle.bmc.util.internal.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
/**
* AwsS3 s3 工具类
*
* @author yunnuo Email: [email protected]
* @since 1.0.0
*/
@Slf4j
public class AwsS3Utils {
/**
* 删除多个文件
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param keySet key
* @return {@link DeleteObjectsResult}
*/
public static DeleteObjectsResult deleteFiles(AmazonS3 amazonS3, String bucket, Set<String> keySet) {
List<DeleteObjectsRequest.KeyVersion> keys = keySet.stream().filter(Objects::nonNull).map(DeleteObjectsRequest.KeyVersion::new).collect(Collectors.toList());
DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucket)
.withKeys(keys)
.withQuiet(false);
return amazonS3.deleteObjects(multiObjectDeleteRequest);
}
/**
* 删除文件
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param key key
*/
public static void deleteFile(AmazonS3 amazonS3,String bucket, String... key){
amazonS3.deleteObject(new DeleteObjectRequest(bucket, S3CommonUtils.formatFilePath(key)));
}
/**
* 删除文件
*
* @param amazonS3 amazon s3
* @param deleteObjectRequest 删除对象请求
*/
public static void deleteFile(AmazonS3 amazonS3, DeleteObjectRequest deleteObjectRequest){
amazonS3.deleteObject(deleteObjectRequest);
}
/**
* 共享文件 7天
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param key key
* @return {@link URL}
*/
public static URL shareFileFor7Day(AmazonS3 amazonS3, String bucket, String... key){
DateTime offset = DateUtil.offsetDay(new Date(), 7);
return shareFile(amazonS3, bucket, offset, key);
}
/**
* 共享文件
* 注意: 使用 Amazon 软件开发工具包，预签名 URL 的最长过期时间为自创建时起 7 天
* @param amazonS3 amazon s3
* @param bucket 桶
* @param expDate 过期日期
* @param key key
* @return {@link URL}
*/
public static URL shareFile(AmazonS3 amazonS3, String bucket, Date expDate, String... key){
GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(bucket, S3CommonUtils.formatFilePath(key));
urlRequest.setExpiration(expDate);
urlRequest.setMethod(HttpMethod.GET);
return amazonS3.generatePresignedUrl(urlRequest);
}
/**
* 下载文件
*
* @param amazonS3 amazon s3
* @param bucketName bucket名称
* @param fileKey 文件key
* @return {@link InputStream}
*/
public static InputStream downloadFile(AmazonS3 amazonS3, String bucketName, String... fileKey) {
GetObjectRequest request = new GetObjectRequest(bucketName, S3CommonUtils.formatFilePath(fileKey));
S3Object response = amazonS3.getObject(request);
return response.getObjectContent();
}
/**
* 获取s3对象
*
* @param amazonS3 amazon s3
* @param bucketName bucket名称
* @param fileKey 文件key
* @return {@link S3Object}
*/
public static S3Object getS3Object(AmazonS3 amazonS3, String bucketName, String... fileKey) {
GetObjectRequest request = new GetObjectRequest(bucketName, S3CommonUtils.formatFilePath(fileKey));
return amazonS3.getObject(request);
}
/**
* 复制文件
*
* @param amazonS3 amazon s3
* @param sourceBucket 源桶
* @param destinationBucket 目地桶
* @param sourceKey 源key
* @param destinationKey 目地key
* @return {@link CopyObjectResult}
*/
public static CopyObjectResult copyFile(AmazonS3 amazonS3, String sourceBucket, String destinationBucket, String sourceKey, String... destinationKey) {
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucket, sourceKey, destinationBucket, S3CommonUtils.formatFilePath(destinationBucket));
return amazonS3.copyObject(copyObjectRequest);
}
/**
* 复制文件
*
* @param amazonS3 amazon s3
* @param copyObjectRequest 复制对象请求
* @return {@link CopyObjectResult}
*/
public static CopyObjectResult copyFile(AmazonS3 amazonS3, CopyObjectRequest copyObjectRequest) {
return amazonS3.copyObject(copyObjectRequest);
}
/**
* 分段上传文件
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param key key
* @param input 文件输入流
* @param objectMetadata 对象元数据
* @return {@link UploadResult}
* @throws InterruptedException 中断异常
*/
public static UploadResult subsectionUploadFile(AmazonS3 amazonS3, String bucket, InputStream input, ObjectMetadata objectMetadata, String... key) throws InterruptedException {
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, S3CommonUtils.formatFilePath(key), input, objectMetadata);
return subsectionUploadFile(amazonS3, putObjectRequest);
}
/**
* 分段上传文件
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param key key
* @param file 文件
* @return {@link UploadResult}
* @throws InterruptedException 中断异常
*/
public static UploadResult subsectionUploadFile(AmazonS3 amazonS3, String bucket, File file, String... key) throws InterruptedException {
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, S3CommonUtils.formatFilePath(key), file);
return subsectionUploadFile(amazonS3, putObjectRequest);
}
/**
* 分段上传文件
*
* @param amazonS3 amazon s3
* @param putObjectRequest 上传对象请求
* @return {@link UploadResult}
* @throws InterruptedException 中断异常
*/
public static UploadResult subsectionUploadFile(AmazonS3 amazonS3, PutObjectRequest putObjectRequest) throws InterruptedException {
TransferManager tm = TransferManagerBuilder.standard()
.withS3Client(amazonS3)
.build();
Upload upload = tm.upload(putObjectRequest);
return upload.waitForUploadResult();
}
/**
* 预签名上传文件默认包日期
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param expiration 过期时间
* @param bundle 包
* @param key key
* @return {@link URL}
*/
public static URL preUploadFileDefaultBundleDate(AmazonS3 amazonS3, String bucket, Date expiration, String bundle, String... key) {
String tempKey = S3CommonUtils.formatFilePath(key);
String filePath = S3CommonUtils.getBundleCurrentDateKey(bundle, tempKey);
return preUploadFile(amazonS3, bucket, expiration, filePath);
}
/**
* 预签名上传文件(oci S3上传会有跨域问题需要用 ObjectStorageClient进行生成)
* AWS S3如果跨域需要运维进行配合允许跨域: AWS S3跨域 配置
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param expiration 过期时间
* @param key key
* @return {@link URL}
*/
public static URL preUploadFile(AmazonS3 amazonS3, String bucket, Date expiration, String... key) {
try {
String filePath = S3CommonUtils.formatFilePath(key);
return amazonS3.generatePresignedUrl(new GeneratePresignedUrlRequest(bucket,
filePath).withExpiration(expiration).withMethod(HttpMethod.PUT));
} catch (Exception e) {
log.warn("Generate preUrl Request is failure :{}", e.getMessage(), e);
throw new RuntimeException("连接S3失败!");
}
}
/**
* 将文件上传到s3默认包日期 eg: bundle(去点)/yyyyMMdd(当前日期)/key
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param file 文件
* @param bundle 包
* @param key key
* @return {@link PutObjectResult}
*/
public static PutObjectResult uploadFileToS3DefaultBundleDate(AmazonS3 amazonS3, String bucket, File file, String bundle, String... key) {
validationParam(bundle, key);
String tempKey = S3CommonUtils.formatFilePath(key);
String filePath = S3CommonUtils.getBundleCurrentDateKey(bundle, tempKey);
return uploadFileToS3(amazonS3, bucket, file, filePath);
}
/**
* 将文件上传到s3默认包日期 eg: bundle(去点)/yyyyMMdd(当前日期)/key
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param fileInput 文件输入流
* @param bundle 包
* @param key key
* @param fileContentType 文件内容类型
* @return {@link PutObjectResult}
*/
public static PutObjectResult uploadFileToS3DefaultBundleDate(AmazonS3 amazonS3, String bucket, InputStream fileInput, Long contentLength, String fileContentType, String bundle, String... key) {
if (!StrUtil.isAllNotBlank(bundle, fileContentType) || key.length == Constants.NUMBER_ZERO) {
throw new RuntimeException("参数为空!");
}
String tempKey = S3CommonUtils.formatFilePath(key);
String filePath = S3CommonUtils.getBundleCurrentDateKey(bundle, tempKey);
return uploadFileToS3(amazonS3, bucket, fileInput, contentLength, fileContentType, filePath);
}
/**
* 将文件上传到s3默认包日期 eg: bundle(去点)/yyyyMMdd(当前日期)/key
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param multipartFile 多部分文件
* @param bundle 包
* @param key key
* @return {@link PutObjectResult}
* @throws IOException ioexception
*/
public static PutObjectResult uploadFileToS3DefaultBundleDate(AmazonS3 amazonS3, String bucket, MultipartFile multipartFile, String bundle, String... key) throws IOException {
validationParam(bundle, key);
String tempKey = S3CommonUtils.formatFilePath(key);
String filePath = S3CommonUtils.getBundleCurrentDateKey(bundle, tempKey);
return uploadFileToS3(amazonS3, bucket, multipartFile, filePath);
}
private static void validationParam(String bundle, String[] key) {
if (StrUtil.isBlank(bundle) || key.length == Constants.NUMBER_ZERO) {
throw new RuntimeException("参数为空!");
}
}
/**
* 将文件上传到s3
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param file 文件
* @param key key
* @return {@link PutObjectResult}
*/
public static PutObjectResult uploadFileToS3(AmazonS3 amazonS3, String bucket, File file, String... key) {
if (Objects.isNull(amazonS3) || Objects.isNull(file) || key.length == Constants.NUMBER_ZERO) {
throw new RuntimeException("参数为空!");
}
String filePath = S3CommonUtils.formatFilePath(key);
PutObjectRequest request = new PutObjectRequest(bucket, filePath, file);
return amazonS3.putObject(request);
}
/**
* 将文件上传到s3
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param multipartFile multipartFile
* @param key key
* @return {@link PutObjectResult}
* @throws IOException ioexception
*/
public static PutObjectResult uploadFileToS3(AmazonS3 amazonS3, String bucket, MultipartFile multipartFile, String... key) throws IOException {
if (Objects.isNull(amazonS3) || Objects.isNull(multipartFile)) {
throw new RuntimeException("参数为空!");
}
return uploadFileToS3(amazonS3, bucket, multipartFile.getInputStream(), multipartFile.getSize(), multipartFile.getContentType(), key);
}
/**
* 将文件上传到s3
*
* @param amazonS3 amazon s3
* @param bucket 桶
* @param fileInput 文件输入流
* @param key key
* @param fileContentType 文件内容类型
* @return {@link PutObjectResult}
*/
public static PutObjectResult uploadFileToS3(AmazonS3 amazonS3, String bucket, InputStream fileInput, Long contentLength, String fileContentType, String... key) {
if (Objects.isNull(amazonS3) || Objects.isNull(fileInput)) {
throw new RuntimeException("参数为空!");
}
String filePath = S3CommonUtils.formatFilePath(key);
PutObjectRequest request = new PutObjectRequest(bucket, filePath, fileInput, null);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(contentLength);
if (StrUtil.isNotBlank(fileContentType)) {
metadata.setContentType(fileContentType);
request.setMetadata(metadata);
}
return amazonS3.putObject(request);
}
/**
* 获取 S3连接 Amazon S3 Compatibility API
*
* @param config {@link S3CommonConfig} 配置
* @return {@link AmazonS3}
*/
public static AmazonS3 s3client(S3CommonConfig config) {
S3BaseConfig baseConfig = BeanUtil.toBean(config, S3BaseConfig.class);
return s3client(baseConfig);
}
/**
* 获取 S3连接 Amazon S3 Compatibility API
*
* @param config {@link S3BaseConfig} 配置
* @return {@link AmazonS3}
*/
public static AmazonS3 s3client(S3BaseConfig config) {
try {
if (Objects.isNull(config.getType()) || Objects.equals(config.getType(), Constants.NUMBER_ZERO)) {
// AWS
return getAmazonS3ByAWS(config);
} else if (Objects.equals(config.getType(), Constants.NUMBER_ONE)) {
// Oracle
return getAmazonS3ByOracle(config);
}
log.warn("s3 configuration is incorrect,config => {}", config);
throw new RuntimeException("连接S3失败!");
} catch (Exception e) {
log.warn("Failed to connect to S3 server e:{}", e.getMessage(), e);
throw new RuntimeException("连接S3失败!");
}
}
/**
* 获取AmazonS3 Amazon S3 Compatibility API
*
* @param config {@link S3BaseConfig} 配置
* @return {@link AmazonS3}
*/
public static AmazonS3 getAmazonS3ByAWS(S3BaseConfig config) {
log.info("S3 配置 走 AWS环境 ");
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
Regions regions = Regions.fromName(config.getRegions());
builder.withRegion(regions);
if (StrUtil.isNotBlank(config.getS3AccessKeyId()) && StrUtil.isNotBlank(config.getS3SecretAccessKey())) {
builder.withCredentials(
new AWSStaticCredentialsProvider(
new BasicAWSCredentials(config.getS3AccessKeyId(), config.getS3SecretAccessKey())));
} else {
builder.withCredentials(
new EC2ContainerCredentialsProviderWrapper());
}
if (Objects.isNull(builder.build())) {
log.warn("connect S3 Server is failure , please check your config param!");
throw new RuntimeException("连接S3失败!");
}
return builder.build();
}
/**
* 获取兼容版本: OCI S3
* 注意: 此版本为兼容版本AmazonS3, 目前遇到的问题生成预签名url不支持跨域, 需要使用
* 参考地址: https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm
*
*
* @param config {@link S3BaseConfig} 配置
* @return {@link AmazonS3}
* @see AmazonS3ClientBuilder
*/
public static AmazonS3 getAmazonS3ByOracle(S3BaseConfig config) {
log.info("S3 配置 走 Oracle环境 ");
// Put the Access Key and Secret Key here
if (!StringUtils.isNoneBlank(config.getS3AccessKeyId(), config.getS3SecretAccessKey())) {
log.warn("S3 配置 走 Oracle环境, ak, sk 不允许配置为空！ config:{}", config);
throw new RuntimeException("连接S3失败!");
}
if (!StringUtils.isNoneBlank(config.getNamespace(), config.getRegions())) {
log.warn("S3 配置 走 Oracle环境, namespace, regions 不允许配置为空！ config:{}", config);
throw new RuntimeException("连接S3失败!");
}
AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider(new BasicAWSCredentials(
config.getS3AccessKeyId(),
config.getS3SecretAccessKey()));
String endpoint = String.format("%s.compat.objectstorage.%s.oraclecloud.com", config.getNamespace(), config.getRegions());
AwsClientBuilder.EndpointConfiguration endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(endpoint, config.getRegions());
return AmazonS3ClientBuilder
.standard()
.withCredentials(credentials)
.withEndpointConfiguration(endpointConfiguration)
.disableChunkedEncoding()
.enablePathStyleAccess()
.build();
}
}