s3 java 断点续传_ceph储存的S3接口实现（支持断点续传）
/**
* Title: S3Manager
* Description: Ceph储存的s3接口实现，参考文档：
*https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/RetrievingObjectUsingJava.html*http://docs.ceph.org.cn/radosgw/s3/* author: xu jun
* date: 2018/10/22*/@Slf4j
@Servicepublic classS3Manager extends StorageManagerBase implements StorageManager {privatefinal UKID ukid;privatefinal S3ClientConfig s3ClientConfig;privatefinal RedisManage redisManage;privateAmazonS3 amazonClient;
@AutowiredpublicS3Manager(UKID ukid, S3ClientConfig s3ClientConfig, RedisManage redisManage) {this.ukid =ukid;this.s3ClientConfig =s3ClientConfig;this.redisManage =redisManage;
}privateAmazonS3 getAmazonClient() {if (amazonClient == null) {
String accessKey=s3ClientConfig.getAccessKey();
String secretKey=s3ClientConfig.getSecretKey();
String endpoint=s3ClientConfig.getEndPoint();
AWSCredentials credentials= newBasicAWSCredentials(accessKey, secretKey);
ClientConfiguration clientConfig= newClientConfiguration();
clientConfig.setProtocol(Protocol.HTTP);
AmazonS3 conn=AmazonS3ClientBuilder.standard()
.withClientConfiguration(clientConfig)
.withCredentials(newAWSStaticCredentialsProvider(credentials))
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, ""))
.withPathStyleAccessEnabled(true)
.build();//检查储存空间是否创建
checkBucket(conn);
amazonClient=conn;
}returnamazonClient;
}
@Overridepublic String uploadFile(byte[] fileData, String extension) {
log.info("Storage s3 api, upload file start");//生成上传文件的随机序号
long fileId =ukid.getGeneratorID();
String fileName=Long.toString(fileId);//储存空间名
String bucketName =s3ClientConfig.getBucketName();
AmazonS3 conn=getAmazonClient();
PutObjectResult result= conn.putObject(bucketName, fileName, new ByteArrayInputStream(fileData), null);
log.info("Storage s3 api, put object result :{}", result);
log.info("Storage s3 api, upload file end, file name:" +fileName);returnfileName;
}
@Overridepublic String uploadAppenderFile(byte[] fileData, String extension) {
log.info("Storage s3 api, upload appender file start");//生成上传文件的随机序号
long ukId =ukid.getGeneratorID();
String fileName=Long.toString(ukId);//储存空间名
String bucketName =s3ClientConfig.getBucketName();
AmazonS3 conn=getAmazonClient();
List partETags = new ArrayList<>();//初始化分片上传
InitiateMultipartUploadRequest initRequest = newInitiateMultipartUploadRequest(bucketName, fileName);
InitiateMultipartUploadResult initResponse=conn.initiateMultipartUpload(initRequest);
String uploadId=initResponse.getUploadId();
ByteArrayInputStream byteArrayInputStream= newByteArrayInputStream(fileData);
Integer contentLength=fileData.length;//文件上传
UploadPartRequest uploadPartRequest = newUploadPartRequest()
.withBucketName(bucketName)
.withKey(fileName)
.withUploadId(uploadId)
.withPartNumber(1)
.withPartSize(contentLength)
.withInputStream(byteArrayInputStream);
UploadPartResult uploadPartResult=conn.uploadPart(uploadPartRequest);try{
byteArrayInputStream.close();
}catch(IOException e) {throwFileCenterExceptionConstants.INTERNAL_IO_EXCEPTION;
}
partETags.add(uploadPartResult.getPartETag());
Integer partNumber=uploadPartResult.getPartNumber();
S3CacheMode cacheMode= newS3CacheMode();
cacheMode.setPartETags(partETags);
cacheMode.setPartNumber(partNumber);
cacheMode.setUploadId(uploadId);
redisManage.set(fileName, cacheMode);
log.info("Storage s3 api, upload appender file end, fileName: {}", fileName);returnfileName;
}
@Overridepublic voiduploadChunkFile(ChunkFileSaveParams chunkFileSaveParams) {
log.info("Storage s3 api, upload chunk file start");
String fileName=chunkFileSaveParams.getFileAddress();
Result result= redisManage.get(fileName);
JSONObject jsonObject=(JSONObject) result.getData();if (jsonObject == null) {throwFileCenterExceptionConstants.CACHE_DATA_NOT_EXIST;
}
S3CacheMode cacheMode= jsonObject.toJavaObject(S3CacheMode.class);
Integer partNumber=cacheMode.partNumber;
String uploadId=cacheMode.getUploadId();
List partETags =cacheMode.partETags;//储存空间名
String bucketName =s3ClientConfig.getBucketName();
AmazonS3 conn=getAmazonClient();
ByteArrayInputStream byteArrayInputStream= newByteArrayInputStream(chunkFileSaveParams.getBytes());
Integer contentLength=chunkFileSaveParams.getBytes().length;
UploadPartRequest uploadPartRequest= newUploadPartRequest()
.withBucketName(bucketName)
.withKey(fileName)
.withUploadId(uploadId)
.withPartNumber(partNumber+ 1)
.withPartSize(contentLength)
.withInputStream(byteArrayInputStream);
UploadPartResult uploadPartResult=conn.uploadPart(uploadPartRequest);
partETags.add(uploadPartResult.getPartETag());
partNumber=uploadPartResult.getPartNumber();try{
byteArrayInputStream.close();
}catch(IOException e) {throwFileCenterExceptionConstants.INTERNAL_IO_EXCEPTION;
}
S3CacheMode cacheModeUpdate= newS3CacheMode();
cacheModeUpdate.setPartETags(partETags);
cacheModeUpdate.setPartNumber(partNumber);
cacheModeUpdate.setUploadId(uploadId);
redisManage.set(fileName, cacheModeUpdate);if (chunkFileSaveParams.getChunk().equals(chunkFileSaveParams.getChunks() - 1)) {//完成分片上传，生成储存对象
CompleteMultipartUploadRequest compRequest = newCompleteMultipartUploadRequest(bucketName, fileName,
uploadId, partETags);
conn.completeMultipartUpload(compRequest);
}
log.info("Storage s3 api, upload chunk file end");
}
@Overridepublic byte[] downloadFile(String fileName) {
log.info("Storage s3 api, download file start");//储存空间名
String bucketName =s3ClientConfig.getBucketName();
AmazonS3 conn=getAmazonClient();
S3Objectobject;if(conn.doesObjectExist(bucketName, fileName)) {object =conn.getObject(bucketName, fileName);
}else{throwFileCenterExceptionConstants.OBJECT_NOT_EXIST;
}
log.debug("Storage s3 api, get object result :{}", object);byte[] fileByte;
InputStream inputStream;
inputStream= object.getObjectContent();try{
fileByte=IOUtils.toByteArray(inputStream);
inputStream.close();
}catch(IOException e) {throwFileCenterExceptionConstants.INTERNAL_IO_EXCEPTION;
}finally{if (inputStream != null) {try{
inputStream.close();
}catch(IOException e) {
log.error(e.getMessage());
}
}
}
log.info("Storage s3 api, download file end");returnfileByte;
}
@Overridepublic byte[] downloadFile(String fileName, long fileOffset, longfileSize) {
log.info("Storage s3 api, download file by block start");//储存空间名
String bucketName =s3ClientConfig.getBucketName();
AmazonS3 conn=getAmazonClient();
S3Objectobject;if(conn.doesObjectExist(bucketName, fileName)) {
GetObjectRequest getObjectRequest= newGetObjectRequest(bucketName, fileName)
.withRange(fileOffset, fileOffset+fileSize);//范围下载。
object =conn.getObject(getObjectRequest);
}else{throwFileCenterExceptionConstants.OBJECT_NOT_EXIST;
}
log.info("Storage s3 api, get object result :{}", object);//读取数据。
byte[] buf;
InputStreamin = object.getObjectContent();try{
buf= inputToByte(in, (int) fileSize);
}catch(IOException e) {throwFileCenterExceptionConstants.INTERNAL_IO_EXCEPTION;
}finally{try{in.close();
}catch(IOException e) {
log.error(e.getMessage());
}
}
log.info("Storage s3 api, download file by block end");returnbuf;
}
@OverridepublicString fileSecret(String filePath) {return null;
}
@OverridepublicString fileDecrypt(String filePath) {return null;
}
@OverridepublicString getDomain() {return null;
}/**
* 检查储存空间是否已创建
*
* @param conn 客户端连接*/
private voidcheckBucket(AmazonS3 conn) {//储存空间名
String bucketName =s3ClientConfig.getBucketName();if(conn.doesBucketExist(bucketName)) {
log.debug("Storage s3 api, bucketName is found:" +bucketName);
}else{
log.warn("Storage s3 api, bucketName is not exist, create it:" +bucketName);
conn.createBucket(bucketName);
}
}/**
* inputStream转byte[]
*
* @param inStream 输入
* @param fileSize 文件大小
* @return 输出
* @throws IOException 异常*/
private static byte[] inputToByte(InputStream inStream, intfileSize) throws IOException {
ByteArrayOutputStream swapStream= newByteArrayOutputStream();byte[] buff = new byte[fileSize];intrc;while ((rc = inStream.read(buff, 0, fileSize)) > 0) {
swapStream.write(buff,0, rc);
}returnswapStream.toByteArray();
}/**
* 调试用的方法，可以在控制台看到io的数据
*
* @param input 输入
* @throws IOException 异常
private static void displayTextInputStream(InputStream input) throws IOException {
// Read the text input stream one line at a time and display each line.
BufferedReader reader = new BufferedReader(new InputStreamReader(input));
String line;
while ((line = reader.readLine()) != null) {
log.info(line);
}
}*/}