AWS S3 学习小结
1.首先，这个是AWS的开发资源使用文档：AWS开发文档，AWS官网 - S3教程
2.我们可以通过AWS Cli和Java Api来操作AWS 的 S3，AWS Cli安装教程：AWS Cli安装
3.Linux下连接S3前，需要先获取到AWS的IAM的accessKey 和secretKey，那么获取方式是：
服务->安全、身份与合规 分组下的 IAM->用户->安全证书->创建访问密钥
然后，
4.获取到了key之后，以下通过AmazonS3来操作S3：
1) 上传文件到S3
public static String uploadToS3(AmazonS3 s3, File tempFile, String remoteFileName, String bucketName) throws IOException {
try {
//上传文件
s3.putObject(new PutObjectRequest(bucketName, remoteFileName, tempFile).withCannedAcl(CannedAccessControlList.PublicRead));
//获取一个request
GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(
bucketName, remoteFileName);
//生成公用的url
URL url = s3.generatePresignedUrl(urlRequest);
System.out.println("=========URL=================\n" + url + "\n============URL=============");
return url.toString();
} catch (AmazonServiceException ase) {
ase.printStackTrace();
} catch (AmazonClientException ace) {
ace.printStackTrace();
}
return null;
}2)下载文件到本地(两种方法自己权衡吧)
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import com.amazonaws.ClientConfiguration;
/**
* @description:
* @create: 2019/4/26 16:24
**/
public class DownloadFile {
/**
* @Title: downFromS3
* @Description: 将文件下载到本地路径
* @param @param remoteFileName 文件名
* @param @param path 下载的路径
* @param @throws IOException 设定文件
* @return void 返回类型
* @throws
*/
public static void downFromS3(AmazonS3 s3, String remoteFileName, String path, String bucketName) throws IOException {
try {
GetObjectRequest request = new GetObjectRequest(bucketName,remoteFileName);
s3.getObject(request,new File(path));
} catch (Exception e) {
e.printStackTrace();
}
}
public static String millToDate(long mills) {
SimpleDateFormat sdf =new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return sdf.format(new Date(Long.valueOf(mills+"")));
}
public static void main(String[] args)
{
ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.setConnectionTimeout(60*60*1000);
clientConfig.setSocketTimeout(60*60*1000);
// (AWS_ACCESS_KEY_ID (or AWS_ACCESS_KEY) and AWS_SECRET_KEY (or AWS_SECRET_ACCESS_KEY)
System.setProperty("aws.accessKeyId","asd");
System.setProperty("aws.secretKey","tTc");
String bucket_name = "buk";
String key_name = "testDir/files.zip";
System.out.format("Downloading %s from S3 bucket %s... on %s\n", key_name, bucket_name, millToDate(System.currentTimeMillis()));
final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withCredentials(DefaultAWSCredentialsProviderChain.getInstance())
//.withClientConfiguration(clientConfig).withPathStyleAccessEnabled(true)
.withRegion(Regions.US_EAST_2).build();
try {
S3Object o = s3.getObject(bucket_name, key_name);
S3ObjectInputStream s3is = o.getObjectContent();
FileOutputStream fos = new FileOutputStream(new File("C:\\Users\\Desktop\\rawfiles.Zip"));
byte[] read_buf = new byte[1024];
int read_len = 0;
while ((read_len = s3is.read(read_buf)) > 0) {
fos.write(read_buf, 0, read_len);
}
s3is.close();
fos.close();
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
} catch (FileNotFoundException e) {
System.err.println(e.getMessage());
System.exit(1);
} catch (IOException e) {
System.err.println(e.getMessage());
System.exit(1);
}
System.out.println("Done!"+ millToDate(System.currentTimeMillis()));
}
}
3) 测试
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Region;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.transfer.TransferManager;
import utils.DownloadFile;
import utils.UploadFile;
import java.io.File;
import java.io.IOException;
import java.util.List;
public class testConnection {
static AmazonS3 s3;
static TransferManager tx;
private static String AWS_ACCESS_KEY = "abc";
private static String AWS_SECRET_KEY = "efg";
static final String bucketName = "hij";
static {
ClientConfiguration config = new ClientConfiguration();
config.setConnectionTimeout(10000);
config.setSocketTimeout(300000);
s3 = new AmazonS3Client(new BasicAWSCredentials(AWS_ACCESS_KEY, AWS_SECRET_KEY), config);
Region usWest2 = Region.getRegion(Regions.US_EAST_1);
s3.setRegion(usWest2);
}
public static void main(String[] args) {
//枚举bucket
List buckets = s3.listBuckets();
for (Bucket bucket : buckets) {
System.out.println("Bucket: " + bucket.getName() + bucket.getOwner());
}
// 上传
try {
UploadFile.uploadToS3(s3, new File("D:\\workplace\\xml_\\i2.xml"), "testFile", bucketName);
} catch (IOException e) {
e.printStackTrace();
}
// 下载
try {
DownloadFile.downFromS3(s3, "test", "C:\\Users\\Desktop\\in.xml", bucketName);
} catch (IOException e) {
e.printStackTrace();
}
}
// 按行读取S3的GZip
public void test13() {
System.setProperty("aws.accessKeyId", "abc");
System.setProperty("aws.secretKey", "asd");
String bucket_name = "test";
String key_name = "data_date=20190814/inst.txt.gz";
System.out.format("Downloading %s from S3 bucket %s...\n", key_name, bucket_name);
final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withCredentials(DefaultAWSCredentialsProviderChain.getInstance()).withRegion(Regions.US_EAST_2).build();
try {
S3Object o = s3.getObject(bucket_name, key_name);
S3ObjectInputStream s3is = o.getObjectContent();
InputStream in = new GZIPInputStream(s3is);
BufferedReader in2=new BufferedReader(new InputStreamReader(in));
String y="";
while((y=in2.readLine())!=null){//一行一行读
System.out.println(y);
}
s3is.close();
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
} catch (FileNotFoundException e) {
System.err.println(e.getMessage());
System.exit(1);
} catch (IOException e) {
System.err.println(e.getMessage());
System.exit(1);
}
}
}
5. 以下通过S3Client来操作S3：
1) 小文件流上传到S3
public void putObject(String bucket, String key, String filePath) throws IOException {
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).build(),
RequestBody.fromByteBuffer(ByteBuffer.wrap(inputStream2ByteArray(filePath))));
}
public void putObject(String bucket, String key, byte[] bytes) {
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).build(),
RequestBody.fromByteBuffer(ByteBuffer.wrap(bytes)));
}
public static byte[] inputStream2ByteArray(String filePath) throws IOException {
try (InputStream in = new FileInputStream(filePath)) {
byte[] data = toByteArray(in);
return data;
}
}2) 分部上传文件流到S3
/**
* Get upload id from s3
*/
public static String getUploadId(String bucketName, String key, Region region){
s3 = S3Client.builder().region(region).build();
// First create a multipart upload and get upload id
CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder()
.bucket(bucketName).key(key)
.build();
CreateMultipartUploadResponse response = s3.createMultipartUpload(createMultipartUploadRequest);
String uploadId = response.uploadId();
return uploadId;
}
/**
* Multipart upload
*/
public static void multipartUpload(String bucketName, String key,
List etags, List completedParts,
String uploadId, byte[] cbuf, int off) {
Region region = Region.US_EAST_2;
s3 = S3Client.builder().region(region).build();
//record upload end tag
ByteBuffer bf = ByteBuffer.wrap(cbuf);
UploadPartRequest uploadPartRequest = UploadPartRequest.builder().bucket(bucketName).key(key)
.uploadId(uploadId)
.partNumber(off).build();
String etag = s3.uploadPart(uploadPartRequest, RequestBody.fromByteBuffer(bf)).eTag();
CompletedPart part = CompletedPart.builder().partNumber(off).eTag(etag).build();
etags.add(etag);
completedParts.add(part);
}
/**
* Complete multipart upload
*/
public static void completeMultipartUpload(String bucketName, String key, List completedParts, String uploadId) {
// Finally call completeMultipartUpload operation to tell S3 to merge all uploaded
// parts and finish the multipart operation.
CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
CompleteMultipartUploadRequest completeMultipartUploadRequest =
CompleteMultipartUploadRequest.builder().bucket(bucketName).key(key).uploadId(uploadId)
.multipartUpload(completedMultipartUpload).build();
s3.completeMultipartUpload(completeMultipartUploadRequest);
} 其中的先后顺序是：getUploadId -> multipartUpload ... -> completeMultipartUpload
注：
1.遇到：AWS error downloading object from S3, “profile file cannot be null”，参考：https://stackoverflow.com/questions/41796355/aws-error-downloading-object-from-s3-profile-file-cannot-be-null
2.遇到：SocketTimeoutException: Read timed out，参考：http://www.unixresources.net/faq/28195217.shtml
3.遇到：S3的Status Code: 404 指的是 bucket 名字写错了
4.遇到：S3的Status Code: 301，那么检查一下Region对不对。
参考：1.AWS S3使用demo
2.S3 java SDK连接
3.命令行管理aws s3
附：
1. https://blog.csdn.net/harryhare/article/details/80710279
2. API 示例查看文档