中意阳光年金 - AWS S3 附件上传
AWS S3 附件上传
- 一、Java 代码调用
-
- 1.上传下载相关代码
- 2.配置文件
- 二、linux 服务器直接上传附件（大批量附件迁移）
-
- 1.linux安装awscli
-
- 1.1 安装awscli
- 1.2 配置awscli
- 1.3 awscli命令上传附件
- 1.4 aws参考网址
- 2. linux安装convmv(附件存在乱码，需要修改编码)
-
- 2.1 安装convmv
- 2.2 Convmv具体用法
一、Java 代码调用
1.上传下载相关代码
- S3Store.java
/*
* Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.gc.annuity.common.util;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.Authenticator;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.Grantee;
import com.amazonaws.services.s3.model.GroupGrantee;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.Permission;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.neusoft.fdframework.core.ServiceException;
import com.neusoft.report.common.lib.StringUtil;
/**
*
* application name： 中意人寿阳光年金系统
* application describing： 此类用于实现S3存储的增、删、改查
* copyright： Copyright © 2017 中意人寿版权所有
* company： generalichina.com
* time： Oct 31, 2017 - 8:58:07 AM
*
* @author [email protected]
* @version $Revision: 1.2 $
*/
public class S3Store {
private static Logger loginfo = LoggerFactory.getLogger(S3Store.class);
private String bucketName;
//private String credentialsFile = "";
private String credFilePath = (S3Store.class.getResource("/").getPath() + "resource/config/credentials");
//private static S3Store instance = new S3Store();
private AmazonS3 s3 = null;
/**
* 分装路径
* @author:jia_yh 20190422
*/
private String loadingpath;
public static synchronized S3Store getInstance() {
return SingletonHolder.instance;
}
public static class SingletonHolder {
private static S3Store instance = new S3Store();
}
public S3Store() {
String configFilePath = "/resource/config/proxy.properties";
ParseConfig config = new ParseConfig();
bucketName = config.getValueByKey(configFilePath, "bucketName");
if(config.getValueByKey(configFilePath, "proxyAccess").equals("true")){
Authenticator.setDefault(new BasicAuthenticator(config
.getValueByKey(configFilePath, "proxyUsername"), config
.getValueByKey(configFilePath, "proxyPassword")));
System.setProperty("https.proxyHost", config.getValueByKey(configFilePath, "https.proxyHost"));
System.setProperty("https.proxyPort", config.getValueByKey(configFilePath, "https.proxyPort"));
System.setProperty("http.proxyHost", config.getValueByKey(configFilePath, "http.proxyHost"));
System.setProperty("http.proxyPort", config.getValueByKey(configFilePath, "http.proxyPort"));
System.setProperty("ftp.proxyHost", config.getValueByKey(configFilePath, "ftp.proxyHost"));
System.setProperty("ftp.proxyPort", config.getValueByKey(configFilePath, "ftp.proxyPort"));
System.setProperty("socks.proxyHost", config.getValueByKey(configFilePath, "socks.proxyHost"));
System.setProperty("socks.proxyPort", config.getValueByKey(configFilePath, "socks.proxyPort"));
}
/*
* 初始化S3服务
*/
AWSCredentials credentials = null;
try {
credentials = new ProfileCredentialsProvider(credFilePath, "default").getCredentials();
} catch (Exception e) {
throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct "
+ "location (resource/config/credentials), and is in valid format.", e);
}
s3 = AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(credentials)).withRegion("cn-north-1").build();
loadingpath = config.getValueByKey(configFilePath, "splitloadingpath");
}
/**
* 上传
*
* @param filePath
* @return
* @author:Jansen Zhang
*/
public HashMap<String, String> uploadObject(File uploadFile, String key) {
//File uploadFile = new File(filePath);
HashMap<String, String> hm = new HashMap<String, String>();
try {
key = this.isPathExit(key);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, uploadFile);
s3.putObject(putObjectRequest);
ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(key));
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
hm.put("size", String.valueOf(objectSummary.getSize()));
hm.put("name", objectSummary.getKey());
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
hm.put("lastModified", sdf.format(objectSummary.getLastModified()));
}
hm.put("bucketName", bucketName);
hm.put("flag", "01");//表示上传成功
hm.put("message", "文件上传至S3服务器成功！");
loginfo.info("文件上传至S3服务器成功，" + uploadFile);
} catch (Exception ase) {
loginfo.error("上传异常文件路径："+uploadFile.getName());
loginfo.error("文件上传至S3服务器异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
hm.put("flag", "02");
hm.put("message", "文件上传连接服务器异常,请联系管理员！");
}
/*catch (AmazonServiceException ase) {
loginfo.error("文件上传至S3服务器异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
loginfo.error("HTTP Status Code: " + ase.getStatusCode());
loginfo.error("AWS Error Code: " + ase.getErrorCode());
loginfo.error("Error Type: " + ase.getErrorType());
loginfo.error("Request ID: " + ase.getRequestId());
hm.put("flag", "02");
hm.put("message", "文件上传连接服务器异常！");
} catch (AmazonClientException ace) {
loginfo.error("文件上传至S3服务器发生严重错误：请查看是否网络故障！");
loginfo.error("Error Message: " + ace.getMessage());
hm.put("flag", "02");
hm.put("message", "文件上传至云服务器连接中断：请重新上传，或联系管理员！");
}*/
return hm;
}
/**
*
* {文件上传}
* 异常时，可以将异常抛出，在业务层做处理
* @param uploadFile
* @param key
* @return
* @author:Brian
*/
public HashMap<String, String> upload(File uploadFile, String key) {
//File uploadFile = new File(filePath);
HashMap<String, String> hm = new HashMap<String, String>();
try {
key = this.isPathExit(key);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, uploadFile);
s3.putObject(putObjectRequest);
ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(key));
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
hm.put("size", String.valueOf(objectSummary.getSize()));
hm.put("name", objectSummary.getKey());
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
hm.put("lastModified", sdf.format(objectSummary.getLastModified()));
}
hm.put("bucketName", bucketName);
hm.put("flag", "01");//表示上传成功
hm.put("message", "文件上传至S3服务器成功！");
loginfo.info("文件上传至S3服务器成功，" + uploadFile);
} catch (AmazonServiceException ase) {
loginfo.error("文件上传至S3服务器异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
throw ase;
}
/*catch (AmazonServiceException ase) {
loginfo.error("文件上传至S3服务器异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
loginfo.error("HTTP Status Code: " + ase.getStatusCode());
loginfo.error("AWS Error Code: " + ase.getErrorCode());
loginfo.error("Error Type: " + ase.getErrorType());
loginfo.error("Request ID: " + ase.getRequestId());
hm.put("flag", "02");
hm.put("message", "文件上传连接服务器异常！");
} catch (AmazonClientException ace) {
loginfo.error("文件上传至S3服务器发生严重错误：请查看是否网络故障！");
loginfo.error("Error Message: " + ace.getMessage());
hm.put("flag", "02");
hm.put("message", "文件上传至云服务器连接中断：请重新上传，或联系管理员！");
}*/
return hm;
}
/**
*
* {方法功能中文描述}
*
* @param uploadFile
* @param key
* @param expirationDate 不能超过七天
* @return
* @author:Brian
*/
public HashMap<String, String> uploadObject(File uploadFile, String key,Date expirationDate) {
HashMap<String, String> res = this.upload(uploadFile, key);
GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(
bucketName, loadingpath+"/"+key);
//设置过期时间
urlRequest.setExpiration(expirationDate);
//生成公用的url
URL url = s3.generatePresignedUrl(urlRequest);
if (url == null) {
throw new ServiceException("can't get s3 file url!");
}
res.put("tempUrl", url.toString());
return res;
}
/**
* {删除文件}
*
* @return 删除成功返回true,失败返回false
* @author:Jansen Zhang
*/
public boolean deleteObject(String filePath) {
boolean reb = true;
try {
//filePath = this.isPathExit(filePath);
//去除对S3服务器的删除操作
//s3.deleteObject(bucketName, filePath);
loginfo.info("删除S3文件成功，" + filePath);
} catch (Exception ase) {
loginfo.error("删除S3文件至异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
reb = false;
}
/*catch (AmazonServiceException ase) {
loginfo.error("删除S3文件至异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
loginfo.error("HTTP Status Code: " + ase.getStatusCode());
loginfo.error("AWS Error Code: " + ase.getErrorCode());
loginfo.error("Error Type: " + ase.getErrorType());
loginfo.error("Request ID: " + ase.getRequestId());
reb = false;
} catch (AmazonClientException ace) {
loginfo.error("文件上传至S3服务器发生严重错误：请查看是否网络故障！");
loginfo.error("Error Message: " + ace.getMessage());
reb = false;
}*/
return reb;
}
/**
* 下载文件
*
* @return 返回输入流
* @author:Jansen Zhang
*/
public InputStream downloadObject(String filePath) {
InputStream retInputStream = null;
try {
filePath = this.isPathExit(filePath);
S3Object object = s3.getObject(new GetObjectRequest(bucketName, filePath));
retInputStream = object.getObjectContent();
loginfo.info("下载S3文件成功，" + filePath);
} catch (Exception ase) {
loginfo.info("下载S3文件路径，" + filePath);
loginfo.error("下载S3文件至异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
}
/*catch (AmazonServiceException ase) {
loginfo.error("下载S3文件至异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
loginfo.error("HTTP Status Code: " + ase.getStatusCode());
loginfo.error("AWS Error Code: " + ase.getErrorCode());
loginfo.error("Error Type: " + ase.getErrorType());
loginfo.error("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
loginfo.error("文件上传至S3服务器发生严重错误：请查看是否网络故障！");
loginfo.error("Error Message: " + ace.getMessage());
}*/
return retInputStream;
}
/**
* 对像是否存在
*
* @return
* @author:Jansen Zhang
*/
public boolean isObjectExit() {
return true;
}
/**
* 列出目录
*
* @return 如果存在返回大于1整数，不存在返回0
* @author:Jansen Zhang
*/
public long objectIsExist(String filePath) {
int i = 0;
try {
//filePath = this.isPathExit(filePath);
ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(filePath));
i = objectListing.getObjectSummaries().size();
} catch (Exception ase) {
loginfo.error("列出S3文件至异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
}
/*catch (AmazonServiceException ase) {
loginfo.error("列出S3文件至异常，可能原因如下:");
loginfo.error("Error Message: " + ase.getMessage());
loginfo.error("HTTP Status Code: " + ase.getStatusCode());
loginfo.error("AWS Error Code: " + ase.getErrorCode());
loginfo.error("Error Type: " + ase.getErrorType());
loginfo.error("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
loginfo.error("文件上传至S3服务器发生严重错误：请查看是否网络故障！");
loginfo.error("Error Message: " + ace.getMessage());
}*/
return i;
}
/**
* 判断目录层级是否包含
*
* @return
* @author:jia_yh
*/
public String isPathExit(String path) {
if(!StringUtil.isEmpty(path) && path.indexOf(loadingpath)==-1){
path = loadingpath+"/"+path;
}
return path;
}
/**
*
* {修改权限为公开只读，并返回公开URL}
*
* @param filePath S3上的文件路径，不用包含最外层拼接目录
* @return
* @author:brave.ye
*/
public String modifyPermissionToPublicRead(String filePath){
filePath = this.isPathExit(filePath);
return modifyACLExistingObject(filePath, GroupGrantee.AllUsers, Permission.Read);
}
/**
*
* {修改已存在文件权限}
*
* @param filePath
* @author:brave.ye
*/
private String modifyACLExistingObject(String filePath, Grantee grantee, Permission permission) {
try {
// Get the existing object ACL that we want to modify.
AccessControlList acl = s3.getObjectAcl(bucketName, filePath);
// Clear the existing list of grants.
// acl.getGrantsAsList().clear();//清除原始权限
/**
* 授予访问权限时，可以将每个被授权者指定为一个 type=value 对，其中 type（类型）为以下值之一：
id – 如果指定的值是 AWS 账户的规范用户 ID
uri – 如果您正在向预定义组授予权限
emailAddress – 如果指定的值是 AWS 账户的电子邮件地址
acl.grantPermission(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl);//根据ID
acl.grantPermission(new EmailAddressGrantee("[email protected]"), Permission.WriteAcp);//根据emailAddress
acl.grantPermission(GroupGrantee.AllUsers, Permission.Read);//根据组 设置公开权限
*/
// Grant a sample set of permissions, using the existing ACL owner for Full Control permissions.
// acl.grantPermission(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl);//根据ID
// acl.grantPermission(new EmailAddressGrantee("[email protected]"), Permission.WriteAcp);//根据emailAddress
// acl.grantPermission(GroupGrantee.AllUsers, Permission.Read);//根据组 设置公开权限
acl.grantPermission(grantee, permission);
// Save the modified ACL back to the object.
s3.setObjectAcl(bucketName, filePath, acl);
URL url = s3.getUrl(bucketName, filePath);
loginfo.info("公开文件的URL为： "+ url.toString());
return url.toString();
} catch (AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
e.printStackTrace();
} catch (SdkClientException e) {
// Amazon S3 couldn't be contacted for a response, or the client
// couldn't parse the response from Amazon S3.
e.printStackTrace();
}
return null;
}
/**
*
* {上传目录文件}
*
* @param directory 源文件目录
* @param s3DirKeyPrefix 上传至S3目录前缀
* @param publicRead 上传目录文件是否公开
* @author:brave.ye
*/
public void uploadDirBatch(File directory, String s3DirKeyPrefix, boolean publicRead){
loginfo.info("directory path: " + directory.getPath());
loginfo.info("s3DirKeyPrefix path: " + s3DirKeyPrefix);
Iterator<File> filesinFolder = FileUtils.iterateFiles(directory, null, true);
// ExecutorService threadPool = Executors.newFixedThreadPool(nThreads);
ExecutorService threadPool = Executors.newCachedThreadPool();//短任务执行的快
while(filesinFolder.hasNext()) {
File file = filesinFolder.next();
loginfo.info("source file path: " + file.getPath());
//s3DirKeyPrefix 截去最后一个'/'，再拼接上源文件去除目录后的路径,增加上路径前缀 sunshineprod
String uploadFilePath = this.isPathExit(s3DirKeyPrefix.substring(0, s3DirKeyPrefix.length()-1) + (file.getPath().replace(directory.getPath(), "").replace("\\", "/")));
loginfo.info("upload s3 file path: " + uploadFilePath);
threadPool.execute(new S3UploadBatch(this, file, uploadFilePath, publicRead));
}
threadPool.shutdown();
}
private class S3UploadBatch implements Runnable{
private S3Store s3Store = null;
private File file;
private String key;
private boolean publicRead;
S3UploadBatch(S3Store s3Store, File file, String key, boolean publicRead){
this.s3Store = s3Store;
this.file = file;
this.key = key;
this.publicRead = publicRead;
}
@Override
public void run() {
s3Store.uploadObject(file, key);
if(publicRead){
s3Store.modifyPermissionToPublicRead(key);
}
}
}
/**
*
* {批量下载s3文件到本地目录(保留文件s3路径目录)}
*
* @param listDir 下载文件名数组 例如：sunshineprod/CLM/DBZZD/DBZZD3//10a9ea2d-7d1a-b557-41d3-61d6f8353bc9.jpeg
* @param targetDir 下载文件的本地目标目录 例如：C:\\Users\\QSNP235\\Desktop
* @author:brave.ye
*/
public void downS3File2Disk(List<String> listDir, String targetDir){
for (int i = 0; i < listDir.size(); i++) {
InputStream inputStream = this.downloadObject(listDir.get(i));
if(inputStream == null){
loginfo.error("The specified key does not exist：{}",listDir.get(i));
continue;
}
String preDir = targetDir;
String[] fileDirArr = listDir.get(i).replace("//", "/").split("/");
for (int j = 0; j < fileDirArr.length - 1; j++) {
preDir = preDir + "\\" + fileDirArr[j];
// loginfo.info(preDir);
File fileDir = new File(preDir);
if (!fileDir.exists()) {
fileDir.mkdirs();
}
}
loginfo.info("文件下载存放路径为："+(preDir + "/" + fileDirArr[fileDirArr.length - 1]).replace("\\", "/").replace("//", "/"));
try {
FileUtil.toDisk(toByteArray(inputStream),preDir + "\\" + fileDirArr[fileDirArr.length - 1]);
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
*
* {批量下载s3文件到本地目录(不保留文件s3路径目录，目录为@后路径)}
*
* @param listDir 下载文件名数组 例如：sunshineprod/CLM/DBZZD/DBZZD1//bebd5ef2-dac3-c6a0-3716-e76453d1631d.jpeg@2021-02-05江文玉211103197011122115
* @param targetDir 下载文件的本地目标目录 例如：C:\\Users\\QSNP235\\Desktop
* @param masterDir 添加一层主目录
* @author:brave.ye
*/
public void downS3File2Disk(List<String> listDir, String targetDir, String masterDir){
for (int i = 0; i < listDir.size(); i++) {
if(listDir.get(i).indexOf("@") < 0){
loginfo.warn("文件路径：{} 不包含分隔符号”@”,请确认...",listDir.get(i));
continue;
}
String preDir = targetDir;
String[] fileDirArr = listDir.get(i).split("@");//文件路径 、保存路径
InputStream inputStream = this.downloadObject(fileDirArr[0]);
if(inputStream == null){
loginfo.error("The specified key does not exist：{}",listDir.get(i));
continue;
}
preDir = preDir + "\\"+masterDir+"\\" + fileDirArr[1];
// loginfo.info(preDir);
File fileDir = new File(preDir);
if (!fileDir.exists()) {
fileDir.mkdirs();
}
String[] fileDirArr2 = fileDirArr[0].replace("//", "/").split("/");
loginfo.info("文件下载存放路径为："+(preDir + "\\" + fileDirArr2[fileDirArr2.length - 1]).replace("\\", "/").replace("//", "/"));
try {
FileUtil.toDisk(toByteArray(inputStream),preDir + "\\" + fileDirArr2[fileDirArr2.length - 1]);
} catch (IOException e) {
loginfo.error("下载文件异常：{}",listDir.get(i));
}
}
}
/**
*
* {inputStream 转 byte[] 用于下载文件到本地}
*
* @param inputStream
* @return
* @throws IOException
* @author:brave.ye
*/
public static byte[] toByteArray(InputStream inputStream) throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
int n = 0;
while (-1 != (n = inputStream.read(buffer))) {
output.write(buffer, 0, n);
}
return output.toByteArray();
}
/**
*
S3Store s3Store = new S3Store();
//上传单个文件
HashMap map = s3Store.uploadObject(new File("C:\\Users\\QSNP235\\Desktop\\help.html"), "app/QAhtml/help.html");
String res = s3Store.modifyPermissionToPublicRead("app/QAhtml/help.html");
//上传目录
s3Store.uploadDirBatch(new File("C:\\Users\\QSNP235\\Desktop\\QAhtml"), "app/QAhtml/", true);
// //下载文件到本地指定目录
// List listDirs = new ArrayList();
// listDirs.add("sunshineprod/CLM/DBZZD/DBZZD43//24c69073-9472-b6a7-ea1b-031c5d1a9581.jpeg");
// s3Store.downS3File2Disk(listDirs, "C:\\Users\\QSNP235\\Desktop");
// //下载文件到本地指定目录
// List listDirs = new ArrayList();
// listDirs.add("sunshineprod/CLM/DBZZD/DBZZD1//bebd5ef2-dac3-c6a0-3716-e76453d1631d.jpeg@2021-02-05江文玉211103197011122115");
// s3Store.downS3File2Disk(listDirs, "C:\\Users\\QSNP235\\Desktop", "sunshineprod");
*
*/
public static void main(String[] args) throws IOException {
S3Store s3Store = new S3Store();
//上传单个文件
HashMap<String, String> map = s3Store.uploadObject(new File("C:\\Users\\QSNP235\\Desktop\\img_qa20.png"), "app/QAhtml/images/img_qa20.png");
String res = s3Store.modifyPermissionToPublicRead("app/QAhtml/images/img_qa20.png");
// //上传目录
// s3Store.uploadDirBatch(new File("C:\\Users\\QSNP235\\Desktop\\QAhtml"), "app/QAhtml/", true);
//
// //下载文件到本地指定目录
// List listDirs = new ArrayList();
// listDirs.add("sunshineprod/CLM/DBZZD/DBZZD43//24c69073-9472-b6a7-ea1b-031c5d1a9581.jpeg");
// s3Store.downS3File2Disk(listDirs, "C:\\Users\\QSNP235\\Desktop");
// //下载文件到本地指定目录
// List listDirs = new ArrayList();
// listDirs.add("sunshineprod/CLM/DBZZD/DBZZD1//bebd5ef2-dac3-c6a0-3716-e76453d1631d.jpeg@2021-02-05江文玉211103197011122115");
// s3Store.downS3File2Disk(listDirs, "C:\\Users\\QSNP235\\Desktop", "sunshineprod");
}
}
2.配置文件
- proxy.properties
#----------------------配置全局代理---------------------------
proxyAccess=true
https.proxyHost=10.0.60.80
https.proxyPort=8080
http.proxyHost=10.0.60.80
http.proxyPort=8080
ftp.proxyHost=10.0.60.80
ftp.proxyPort=8080
socks.proxyHost=10.0.60.80
socks.proxyPort=8080
proxyUsername=qtmp003
proxyPassword=Aa123456
#--封装路径（测试环境）
splitloadingpath=sunshineprodtest
bucketName=gc-ygnj-t01
- credentials
# Move this credentials file to (~/.aws/credentials)
# after you fill in your access and secret keys in the default profile
# WARNING: To avoid accidental leakage of your credentials,
# DO NOT keep this file in your source directory.
[default]
aws_access_key_id=AKIA5LANYIP67Z25DIWE
aws_secret_access_key=XBdJ6esXa8R8EtcE159+uRuntAfUiOZs/xXQj0uk
二、linux 服务器直接上传附件（大批量附件迁移）
1.linux安装awscli
1.1 安装awscli
aws help查看是否已经安装，按q退出查看连接你的Linux服务器，按照以下步骤操作：
安装pip
yum -y install python-pip
查看是否已经安装了python-pip，直接运行
pip -v
更新
pip install --upgrade pip
安装awscli
pip install awscli
1.2 配置awscli
初始化配置
就是 java 代码中的 credentials 中的配置
aws configure
AWS Access Key ID [None]: *************
AWS Secret Access Key [None]: *************************
Default region name [None]: cn-north-1
Default output format [None]:overview
| 中国 (北京) | cn-north-1 | autoscaling.cn-north-1.amazonaws.com.cn | HTTP and HTTPS |
|---|
1.3 awscli命令上传附件
设置代理
$ export HTTP_PROXY=http://username:[email protected]:1234
$ export HTTPS_PROXY=http://username:[email protected]:5678
上传操作命令：
查看某一服务器下内容
aws s3 ls s3://***/
复制单个文件到aws上去
#例如：
#aws s3 cp filename s3://bucketname/path
aws s3 cp ./*** s3://***/***
上传文件夹：
aws s3 sync /home/files/S3FileUpload/ s3://bucketname/files/
注：上传文件夹时最外层目录不会同步
1.4 aws参考网址
更多的帮助信息详见AWS官网：
http://docs.aws.amazon.com/zh_cn/cli/latest/userguide/cli-chap-welcome.html
关于AWS区域详见AWS官网：
http://docs.aws.amazon.com/zh_cn/general/latest/gr/rande.html
2. linux安装convmv(附件存在乱码，需要修改编码)
2.1 安装convmv
#查看是否已安装convmv
convmv –help
#安装convmv
yum -y install convmv
2.2 Convmv具体用法
convmv -f 源编码 -t 新编码 [选项] 文件名
常用参数：
-r 递归处理子文件 夹
–notest 真正进行操作，请注意在默认情况下是不对文件 进行真实操作的，而只是试验。
–list 显示所有支持的编码
–unescap 可以做一下转义，比如把%20变成空格
例如：convmv -f UTF-8 -t GBK --notest filename
#需要修改编码：iso-8859-1 转换成UTF-8
#先转成GBK:
convmv -f iso-8859-1 -t GBK --notest filename
#再GBK转成UTF-8:
Convmv -f GBK -t UTF-8 --notest filename