amazonS3文件操作知识点汇总

• 1.文件上传时，对于不存在路径会是啥结果？

如果上传的存储桶不存在，会报错。如果是存储桶下面的文件夹不存在，则会创建一个并保存进去你上传的文件。第二次上传同样的路径，应为已经存在，则直接保存。如果上传的路径的 一级目录 前面多了斜杠“/”，也会报错。举个例子：
指定的文件上传的key为：/temp/20200723/小小的船-2129566950.pptx，则就会报错。

The specified key does not exist. (Service: Amazon S3; Status Code: 404; Error Code: NoSuchKey; Request ID: DE8C45718259ACD7; 

• 2.amazonS3能够实现文件操作

上传，下载，复制，删除，但是没有直接的移动操作。并且通过在进入amazonS3进行配置生命周期的规则支持后，就可以实现对上传文件的过期时间的设置。这个过期时间的实现必须的前提条件就是要开启对存储桶指定目录的生命周期。如下图是对temp目录设置好的生命周期规则。其中前面这个transient_rule即是ruleId。

• 3.剩下一些工具类的实现

package com.util;

import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;

import java.io.*;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

/**
 * AmazonS3文件操作工具类
 *
 * @author zhanglifeng
 * @date 2020-07-01
 */
public class AmazonS3Util {
    private static final Logger LOGGER = LoggerFactory.getLogger(AmazonS3Util.class);
    /**
     * access_key_id 你的亚马逊S3服务器访问密钥ID
     */
    private static final String ACCESS_KEY = "AIEKIAF3TXCLQT6M7A7L";
    /**
     * secret_key 你的亚马逊S3服务器访问密钥
     */
    private static final String SECRET_KEY = "3wyLbaxZ62XnMMz517IB36c63jeRev6e8HhxemSS";
    /**
     * end_point 你的亚马逊S3服务器连接路径和端口(新版本不再需要这个,直接在创建S3对象的时候根据桶名和Region自动获取)
     *
     * 格式: https://桶名.s3-你的Region名称.amazonaws.com
     * 示例: https://xxton.s3-cn-north-1.amazonaws.com
     */
    //private static final String END_POINT = "https://xxton.s3-cn-north-1.amazonaws.com";
    /**
     * bucketname 你的亚马逊S3服务器创建的桶名
     */
    private static final String S3_BUCKET_NAME = "media.fenglizhang.com";

    /**
     * 创建访问凭证对象
     */
    private static final BasicAWSCredentials awsCreds = new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY);
    /**
     * 创建s3对象
     */
    private static final AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
            //设置服务器所属地区
            .withRegion(Regions.US_WEST_1)
            .build();

    /**
     * 经过测试，文件上传的方法这个要比下面的uploadToS3的快。
     *
     * @param key          文件在bucket中的存储文件名
     * @param env          当前项目的环境：dev,test,stage,prod
     * @param filePath     待上传的文件存放位置
     * @param s3CdnBaseUrl cdn在不同环境的url
     * @return
     */
    public static String uploadToS3Fast(String key, String env, String filePath, String s3CdnBaseUrl) {
        long startTime = System.currentTimeMillis();
        S3Client client = S3Client.builder().region(Region.US_WEST_1).credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY))).build();
        String bucketName = getS3BucketName(env);
        software.amazon.awssdk.services.s3.model.PutObjectRequest putObjectRequest = software.amazon.awssdk.services.s3.model.PutObjectRequest.builder().bucket(bucketName).key(key).build();
        client.putObject(putObjectRequest, Path.of(filePath));
        String downloadUrl = String.format("https://%s/%s", s3CdnBaseUrl, key);
        long endTime = System.currentTimeMillis();
        LOGGER.info("上传到S3耗时：{}", endTime - startTime);
        return downloadUrl;
    }


    /**
     * @param key          文件在bucket中的存储文件名
     * @param env          当前项目的环境：dev,test,stage,prod
     * @param filePath     待上传的文件存放位置
     * @param s3CdnBaseUrl cdn在不同环境的url
     * @return
     */
    public static String uploadToS3(String key, String env, String filePath, String s3CdnBaseUrl) {
        long startTime = System.currentTimeMillis();
        String bucketName = getS3BucketName(env);
        PutObjectRequest request = new PutObjectRequest(bucketName, key, new File(filePath));
        s3Client.putObject(request);
        String downloadUrl = String.format("https://%s/%s", s3CdnBaseUrl, key);
        long endTime = System.currentTimeMillis();
        LOGGER.info("上传到S3耗时：{}", endTime - startTime);
        return downloadUrl;
    }


    /**
     * @param key            文件在存储桶中的目录
     * @param env            当前项目的环境：dev,test,stage,prod
     * @param targetFilePath 缓存到本地的文件名
     */
    public static void amazonS3Download(String key, String env, String targetFilePath) {
        long startTime = System.currentTimeMillis();

        String bucketName = getS3BucketName(env);
        S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key));
        if (object != null) {
            LOGGER.info("Content-Type: " + object.getObjectMetadata().getContentType());
            InputStream input = null;
            FileOutputStream fileOutputStream = null;
            byte[] data = null;
            try {
                //获取文件流
                input = object.getObjectContent();
                data = new byte[input.available()];
                int len = 0;
                fileOutputStream = new FileOutputStream(targetFilePath);
                while ((len = input.read(data)) != -1) {
                    fileOutputStream.write(data, 0, len);
                }
                LOGGER.info("下载文件成功");
            } catch (IOException e) {
                LOGGER.error("文件流转换发生异常：{}", e);
            } finally {
                try {
                    if (fileOutputStream != null) {
                        fileOutputStream.flush();
                        fileOutputStream.close();
                    }
                    if (input != null) {
                        input.close();
                    }
                    long endTime = System.currentTimeMillis();
                    LOGGER.info("amazonS3Download方法从S3上下载文件耗时：{}", endTime - startTime);
                } catch (IOException e) {
                    LOGGER.error("关闭文件流发生异常：{}", e);
                }
            }
        }
    }


    private static String getS3BucketName(String env) {
        String bucketName;
        if ("prod".equals(env)) {
            bucketName = S3_BUCKET_NAME;
        } else if ("dev".equals(env)) {
            bucketName = String.format("%s.%s", env, S3_BUCKET_NAME);
        } else {
            bucketName = String.format("%s.%s", "stage", S3_BUCKET_NAME);
        }
        return bucketName;
    }

    /**
     * @param url          能够通过浏览器打开的资源链接。比如：https://cdn.fenglizhang.com/cw/1495159251-0000.png  
     * @param saveFile     要保存的文件地址
     * @param restTemplate 需要注入实例的rest客户端
     */
    public static void downloadFileFromS3(String url, File saveFile, RestTemplate restTemplate) {
        try {
            HttpHeaders headers = new HttpHeaders();
            HttpEntity<Resource> httpEntity = new HttpEntity<>(headers);
            ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.GET, httpEntity, byte[].class);
            FileOutputStream fos = new FileOutputStream(saveFile);
            fos.write(response.getBody());
            fos.flush();
            fos.close();
        } catch (Exception e) {
            LOGGER.error(String.format("downloadFile url {} exception", url), e);
        }
    }

    /**
     * 本复制文件的方法适用于在同一个bucket中
     *
     * @param sourceKey      Source object key
     * @param env            当前应用的环境
     * @param destinationKey Destination object key
     */
    public static String copyObjectFromS3ToS3(String sourceKey, String env, String destinationKey, String s3CdnBaseUrl) {
        String bucketName = getS3BucketName(env);
        // Copy the object into a new object in the same bucket.
        CopyObjectRequest copyObjRequest = new CopyObjectRequest(bucketName, sourceKey, bucketName, destinationKey);
  /*      ObjectMetadata objectMetadata = new ObjectMetadata();
        objectMetadata.setExpirationTime();
        objectMetadata.setExpirationTimeRuleId();
        copyObjRequest.setNewObjectMetadata(objectMetadata);*/
        CopyObjectResult copyObjectResult = s3Client.copyObject(copyObjRequest);
        return String.format("https://%s/%s", s3CdnBaseUrl, destinationKey);
    }

    /**
     * 上传文件到暂存区并且设置文件的过期时间
     * @param key          文件在bucket中的存储文件名
     * @param env          当前项目的环境：dev,test,stage,prod
     * @param filePath     待上传的文件存放位置
     * @param s3CdnBaseUrl cdn在不同环境的url
     * @return
     */
    public static String uploadToS3TransientAndSetExpiredTime(String key, String env, String filePath, String s3CdnBaseUrl) {
        long startTime = System.currentTimeMillis();
        String bucketName = getS3BucketName(env);
        File file = new File(filePath);
        PutObjectRequest request = new PutObjectRequest(bucketName, key, file);
        String downloadUrl = null;
        int index = filePath.lastIndexOf(".");
        String contentType = String.format("%s/%s", "application", filePath.substring(index + 1));
        InputStream inputStream = null;
        try {
            inputStream = new FileInputStream(file);

            ObjectMetadata objectMetadata = new ObjectMetadata();
            // 必须设置ContentLength
            objectMetadata.setContentLength(file.length());
            objectMetadata.setExpirationTimeRuleId("transient_rule");
            objectMetadata.setContentType(contentType);
            request.setMetadata(objectMetadata);

            s3Client.putObject(bucketName, key, inputStream, objectMetadata);
            downloadUrl = String.format("https://%s/%s", s3CdnBaseUrl, key);
            long endTime = System.currentTimeMillis();
            LOGGER.info("上传到S3耗时：{}", endTime - startTime);

        } catch (FileNotFoundException e) {
            LOGGER.error(e.getMessage());
        } finally {
            try {
                inputStream.close();
            } catch (IOException e) {
                LOGGER.error("inputStream关闭异常，异常信息：{}",e.getMessage());
            }
        }
        return downloadUrl;
    }


    /**
     * 遍历存储桶指定文件夹下的所有文件路径
     *
     * @param s3
     * @param bucket
     * @param prefix
     * @return
     */
    public static List<String> listKeysInDirectory(S3Client s3, String bucket, String prefix) {
        String delimiter = "/";
        if (!prefix.endsWith(delimiter)) {
            prefix += delimiter;
        }
        // Build the list objects request
        software.amazon.awssdk.services.s3.model.ListObjectsV2Request listReq = software.amazon.awssdk.services.s3.model.ListObjectsV2Request.builder()
                .bucket(bucket)
                .prefix(prefix)
                .delimiter(delimiter)
                .maxKeys(1)
                .build();

        ListObjectsV2Iterable listRes = s3.listObjectsV2Paginator(listReq);
        List<String> keyList = new ArrayList<>();
        final String folder = prefix;
        listRes.contents().stream()
                .forEach(content -> {
                    if (!folder.equals(content.key())) {
                        keyList.add(content.key());
                    }
                });
        return keyList;
    }

    /**
     * 删除指定目录下的所有文件
     *
     * @param env        环境
     * @param folderPath
     */
    public static void deleteS3Folder(String env, String folderPath) {
        String bucketName = getS3BucketName(env);
        S3Client s3 = S3Client.builder().
                region(Region.US_WEST_1).
                credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)))
                .build();
        ArrayList<ObjectIdentifier> to_delete = new ArrayList<ObjectIdentifier>();
        List<String> object_keys = listKeysInDirectory(s3, bucketName, folderPath);
        if (null == object_keys || object_keys.size() == 0) {
            return;
        }
        for (String k : object_keys) {
            to_delete.add(ObjectIdentifier.builder().key(k).build());
        }
        try {
            software.amazon.awssdk.services.s3.model.DeleteObjectsRequest dor = software.amazon.awssdk.services.s3.model.DeleteObjectsRequest.builder()
                    .bucket(bucketName)
                    .delete(Delete.builder().objects(to_delete).build())
                    .build();
            DeleteObjectsResponse response = s3.deleteObjects(dor);
            while (!response.sdkHttpResponse().isSuccessful()) {
                Thread.sleep(100);
            }
        } catch (S3Exception | InterruptedException e) {
            LOGGER.error(e.getMessage());
        }
        LOGGER.info("delete folder successfully--->" + folderPath);
    }
}

• 4.amazon上面工具类的依赖

<!-- aws依赖  -->
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>s3</artifactId>
            <version>2.10.4</version>
            <exclusions>
                <exclusion>
                    <groupId>org.apache.httpcomponents</groupId>
                    <artifactId>httpclient</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>ses</artifactId>
            <version>2.10.4</version>
            <exclusions>
                <exclusion>
                    <groupId>org.apache.httpcomponents</groupId>
                    <artifactId>httpclient</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-s3</artifactId>
            <version>1.11.233</version>
        </dependency>