Minio 批量下载文件，并压缩文件，使用sftp上传文件
公司要求使用minio下载文件，并且通过将文件压缩后上传到不同网络的资源服务器,从minio下载文件，目前只会从一个桶(只包含文件不含目录)中下载纯文件，还没有实现桶中包含文件和文件目录的功能，还有待优化
首先在pom.xml中加入相应的依赖
<dependency>
<groupId>com.jcraftgroupId>
<artifactId>jschartifactId>
<version>0.1.44-1version>
dependency>
<dependency>
<groupId>io.miniogroupId>
<artifactId>minioartifactId>
<version>${minio.version}version>
dependency>
具体代码如下:
package com.xxx.xxxx.xxxxx.dto.apply;
import cn.hutool.core.io.FastByteArrayOutputStream;
import com.xxx.xxx.xxx.exception.EcnyTransException;
import com.xxx.xxx.xxx.exception.WalletTransError;
import io.lettuce.core.metrics.MicrometerCommandLatencyRecorder;
import io.minio.*;
import io.minio.messages.Item;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
/**
* @author Liang Guo
* @version 1.0
* @date 2022/5/23 15:21
* @describe 测试从minio 桶下载数据
*/
public class downloadFileFromMinioTest {
@Test
public void download() {
MinioClient minioClient = MinioClient.builder().endpoint("http://xx.x.xxx.xxx:9000").credentials("xxxxx", "xxxxx").build();
try {
//桶名 test
boolean ofwallet = minioClient.bucketExists(BucketExistsArgs.builder().bucket("test").build());
// 如果桶存在
if (ofwallet) {
//查询桶中的所有我呢见
Iterable<Result<Item>> itemLists =
minioClient.listObjects(ListObjectsArgs.builder().bucket("test").recursive(false).build());
for (Result<Item> itemList : itemLists) {
Item item = itemList.get();
downloadFile(item, minioClient);
}
}
} catch (Exception e) {
}
}
/**
* item 桶中的对象
*/
private void downloadFile(Item item, MinioClient minioClient) {
try {
GetObjectArgs build = GetObjectArgs.builder().bucket("test").object(item.objectName()).build();
//获取文件对象
GetObjectResponse objectResponse = minioClient.getObject(build);
byte[] buf = new byte[Integer.parseInt(String.valueOf(item.size())) + 1]; // 指定长度
FastByteArrayOutputStream fbaos = new FastByteArrayOutputStream();
int len = 0;
while ((len = objectResponse.read(buf)) != -1) {
fbaos.write(buf, 0, len);
}
fbaos.flush(); //刷新
// 指定目录创建文件
File fileDir = new File("/home/download");
if (!fileDir.exists() && !fileDir.isDirectory()) {
fileDir.mkdir();
}
File file = new File( "/home/download/" + item.objectName());
if (item.objectName().contains("/") || item.objectName().contains("\\")) {
file.mkdir();
}
if (!file.exists()) {
file.createNewFile();
}
// 写文件
OutputStream outputStream = new FileOutputStream(file);
outputStream.write(fbaos.toByteArray());
outputStream.flush();
fbaos.close(); // 关流
outputStream.close();
} catch (Exception e) {
throw new EcnyTransException(WalletTransError.SIGN_ECIF_NETWORK_FAIL);
}
}
}
压缩文件:
package com.xxx.xxx.xxx.dto.apply;
import com.xxx.xxx.xxx.service.IGenerateCodeService;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.*;
import java.net.URL;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/**
* @author Liang Guo
* @version 1.0
* @date 2022/5/19 15:07
*/
public class MakeDirTest {
@Autowired
private IGenerateCodeService generateCodeService;
private String PARTNER_NO = "45612346";
/**
* 复制文件
* @throws IOException
*/
@Test
public void makeDir() throws IOException {
try {
//获取file文件夹的相对路径
URL url = getClass().getClassLoader().getResource("file/");
File file = new File(url.getPath());
//判断该文件是否是一个文件夹
if (file != null && file.isDirectory()) {
//将文件夹中的所有文件赋值给数组
File[] listFiles = file.listFiles();
for (File listFile : listFiles) {
System.out.println(url);
File accountOpen = new File(url.getPath() + "/accountOpen");
if (!accountOpen.exists()) {
accountOpen.mkdir();
}
byte[] buf = new byte[1024];
FileInputStream fileInputStream = new FileInputStream(listFile);
FileOutputStream fos = new FileOutputStream(accountOpen.getCanonicalPath()+"/2.txt");
int len = 0;
while ((len = fileInputStream.read(buf)) != -1){
fos.write(buf);
}
//创建一个用于操作文件的字节输出流对象，创建就必须明确数据存储目的地
//刷新并关闭流
fos.flush();
fos.close();
fileInputStream.close();
}
}
} catch (Exception e) {
throw new IOException("复制文件失败",e.getCause());
}
}
/**
* 复制一个文件夹中的内容，并且重命名文件夹
*/
@Test
public void mkdir1() throws IOException {
try {
// 获取/home/downLoad文件夹的路径
File file = new File("/home/download");
//判断该文件是否是一个文件夹，文件不做处理
if (file != null && file.isDirectory()) {
File accountOpen = new File("/home/accountopen");
// 找到文件夹中的所有信息 赋值给数组
File[] listFiles = file.listFiles();
for (File listFile : listFiles) {
//判断文件是否为文件夹
if (listFile.isDirectory()){
//是文件夹，再往下一级查找文件,递归调用 本方法
mkdir1();
} else {
//不是文件夹
// 新建一个accountOpen的文件夹
if (!accountOpen.exists()) {
accountOpen.mkdir();
}
byte[] buff = new byte[listFiles.length];
FileInputStream fileInputStream = new FileInputStream(listFile);
FileOutputStream fos = new FileOutputStream(accountOpen.getCanonicalPath() + "/" + listFile.getName());
int len = 0;
while((len = fileInputStream.read(buff))!= -1){
fos.write(buff);
}
fos.flush();
fos.close();
fileInputStream.close();
}
}
}
} catch (Exception e) {
throw new IOException("",e.getCause());
}
}
/**
* 压缩文件
*/
@Test
public void zipFile() throws IOException {
for(int i = PARTNER_NO.length(); i < 15; i++){
PARTNER_NO = "0" + PARTNER_NO;
}
String zipName = PARTNER_NO + "_001_" + "gdgsar02" + ".zip"; //压缩文件名字
String zipUrl = "/home/sftp/"; // FileOutputStream(zipUrl + zipName) 创建zip压缩文件的路径以及名称 压缩包存放的地方
File file1 = new File(zipUrl);
if (file1 != null && !file1.isDirectory()) {
file1.mkdir();
}
ZipOutputStream zipOutputStream = new ZipOutputStream(new FileOutputStream(zipUrl + zipName));//设置压缩文件名字
File file = new File("/home/accountopen/");
zip(zipOutputStream,file, "");
zipOutputStream.close();
//删除 accountopen文件加
deleteFile(file);
}
/**
* 删除文件
* @param file
*/
private void deleteFile(File file) {
if (file == null || !file.exists()) {
System.out.println("文件删除失败,请检查文件是否存在以及文件路径是否正确");
}
File[] files = file.listFiles();
for (File f : files) {
if (f.isDirectory()) {
deleteFile(f);
} else {
f.delete();
}
}
file.delete();
System.out.println("删除成功" + file.getName());
}
private void zip(ZipOutputStream zipOutputStream, File file, String base) throws IOException {
String baseUrl = "accountopen/"; // 压缩包内部目录
if (null != file && file.isDirectory()) { //如果该文件对象是一个目录
File[] files = file.listFiles();
zipOutputStream.putNextEntry(new ZipEntry(baseUrl));
for (File file1 : files) {
zip(zipOutputStream, file1, file1.getName());
}
} else {
zipOutputStream.putNextEntry(new ZipEntry(baseUrl + base)); //压缩包内的目录结构
FileInputStream fileInputStream = new FileInputStream(file);
int b;
while((b = fileInputStream.read())!=-1){
zipOutputStream.write(b);
}
fileInputStream.close();
}
}
}
用sftp上传文件，上传文件待修改，本次只是记录，还未测试
package com.xxx.xxx.xxx.dto.apply;
import com.jcraft.jsch.*;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.Properties;
/**
* @author Liang Guo
* @version 1.0
* @date 2022/5/24 9:39
*/
public class SFTPPutFileTest {
private static final String hostname = "xx.x.xxx.xxx"; //ip地址
private static final String port = "22"; //端口号
private static final String sshUser = "xxxxx"; //用户名
private static final String sshPass = "xxxxx"; //密码
@Test
public void putFileToServer() throws Exception {
File uploadFile = new File("/home/sftp/xxxx_001_gdgsar02.zip");
JSch jSch = new JSch();
String filePath = "/home/sftp/";
Session session = null;
ChannelSftp sftp = null;
try {
session = jSch.getSession(sshUser, hostname, Integer.parseInt(port));
session.setPassword(sshPass);
Properties sshConfig = new Properties();
sshConfig.put("StrictHostKeyChecking", "no");
session.setConfig(sshConfig);
session.connect();
// 获取上传的管道
Channel channel = session.openChannel("sftp");
channel.connect();
sftp = (ChannelSftp) channel;
//发送文件、
sftp.cd(filePath); //进入指定目录 ,文件保存目录
System.out.println("-------上传文件-------:"+ uploadFile);
sftp.put(new FileInputStream(uploadFile), uploadFile.getName()); //读取要上传的文件并上传
System.out.println("-------上传完成-------:"+ filePath);
} catch (JSchException | FileNotFoundException | SftpException e){
throw e;
}
}
}
sftp上传文件，已测试
package com.xxx.xxx.xxx.dto.apply;
import com.jcraft.jsch.*;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.Properties;
/**
* @author Liang Guo
* @version 1.0
* @date 2022/5/24 9:39
*/
public class SFTPPutFileTest {
private static final String hostname = "xx.x.xxx.xxx";
private static final String port = "22";
private static final String sshUser = "root";
private static final String sshPass = "xxxxxx";
@Test
public void putFileToServer() throws Exception {
File uploadFile = new File("/home/sftp/00000PT00000308_001_gdgsar02.zip");
JSch jSch = new JSch();
String filePath = "/home/sftp/";
Session session = null;
ChannelSftp sftp = null;
try {
session = jSch.getSession(sshUser, hostname, Integer.parseInt(port));
session.setPassword(sshPass);
Properties sshConfig = new Properties();
sshConfig.put("StrictHostKeyChecking", "no");
session.setConfig(sshConfig);
session.connect();
// 获取上传的管道
Channel channel = session.openChannel("sftp");
channel.connect();
sftp = (ChannelSftp) channel;
//判断服务器是否有该目录 /home/sftp
SftpATTRS lstat = sftp.lstat(filePath);
//if (!lstat.isDir()) { //如果不是目录
// sftp.mkdir(filePath); //创建目录
//} 待完善
//发送文件、
sftp.cd(filePath); //进入指定服务器目录 ,文件保存目录
System.out.println("-------上传文件-------:"+ uploadFile);
sftp.put(new FileInputStream(uploadFile), uploadFile.getName()); //读取要上传的文件并上传
System.out.println("-------上传完成-------:"+ filePath);
} catch (JSchException | FileNotFoundException | SftpException e){
throw e;
}
}
}
完成